Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.285616071,
|
||||
"end_mono_ns": 208628026400377,
|
||||
"end_wall_ns": 1783877055152827630,
|
||||
"start_mono_ns": 208566740784306,
|
||||
"start_wall_ns": 1783876993867211598
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 39.26863328358488,
|
||||
"min": 4.784946039540869,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1786.641047016019,
|
||||
"min": 38.60557498410344,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.484334594,
|
||||
"end_mono_ns": 208696038541120,
|
||||
"end_wall_ns": 1783877123164968015,
|
||||
"start_mono_ns": 208634554206526,
|
||||
"start_wall_ns": 1783877061680633802
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 44.45809762976243,
|
||||
"min": 4.488573606239676,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1778.0322160106152,
|
||||
"min": 47.90777899324894,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.15579710144927536, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 92,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 26.818138737,
|
||||
"end_mono_ns": 208796704305524,
|
||||
"end_wall_ns": 1783877223830732816,
|
||||
"start_mono_ns": 208769886166787,
|
||||
"start_wall_ns": 1783877197012593929
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.15579710144927536,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 4.6,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 43,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 92,
|
||||
"finite_n": 92,
|
||||
"max": 40.052486527445325,
|
||||
"min": 11.32848871666498,
|
||||
"missing_n": 184,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 92,
|
||||
"finite_n": 92,
|
||||
"max": 6538.087566004833,
|
||||
"min": 66.57706701662391,
|
||||
"missing_n": 184,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp1_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 14174,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8709 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.24609375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/confirm-1-anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.5 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.468118415,
|
||||
"end_mono_ns": 208763972089008,
|
||||
"end_wall_ns": 1783877191098516077,
|
||||
"start_mono_ns": 208702503970593,
|
||||
"start_wall_ns": 1783877129630397769
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 44.40597959039696,
|
||||
"min": 4.490311023564716,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1776.1785559996497,
|
||||
"min": 49.54166599782184,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2781473-1783876972093434233.jsonl","encoded_records":14174,"written_records":14174,"dropped_records":0,"last_step_index":14173,"checkpoint_wall_ns":1783877224563096809,"flush_interval_seconds":1.0,"final":true}
|
||||
@@ -0,0 +1,686 @@
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339]
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339]
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8709, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:11 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:21 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:23 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:35361 backend=nccl
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:23 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:24 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:24 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:25 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:25 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:25 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:25 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1288.56 GiB.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:25 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:12, 1.16it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:12, 1.13it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:11, 1.13it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:03<00:10, 1.11it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:04<00:09, 1.10it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:05<00:09, 1.09it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:06<00:08, 1.09it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:07<00:07, 1.10it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:08<00:06, 1.11it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:09<00:05, 1.11it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:09<00:04, 1.12it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:10<00:03, 1.12it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:11<00:02, 1.12it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:12<00:01, 1.11it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:13<00:00, 1.11it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.42it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.16it/s]
|
||||
(EngineCore pid=2781473)
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:39 [default_loader.py:430] Loading weights took 13.77 seconds
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:39 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:40 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 14.432905 seconds
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:43 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/f4a50989f8/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:43 [backends.py:1148] Dynamo bytecode transform time: 3.54 s
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:46 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.334 s
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:46 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/fe5a82dbe929018b7aea7dee05b5cd31a21fe2682aca78ee4cbf2b37c8a086d6/rank_0_0/model
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:46 [monitor.py:53] torch.compile took 6.27 s in total
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:46 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:46 [monitor.py:81] Initial profiling/warmup run took 0.24 s
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:47 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_worker.py:508] Available KV cache memory: 29.43 GiB
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [kv_cache_utils.py:2146] GPU KV cache size: 321,456 tokens
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:49 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2781473) 2026-07-12 17:22:49,532 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2781473) 2026-07-12 17:22:49,575 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2781473)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▊ | 2/7 [00:00<00:00, 11.65it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 4/7 [00:00<00:00, 11.68it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:00<00:00, 9.25it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 9.30it/s]
|
||||
(EngineCore pid=2781473)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 12.61it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 12.94it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 12.99it/s]
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:51 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.10 GiB
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:51 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.02 GiB (15.7%).
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:51 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:51 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.53 s (compilation: 6.27 s)
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:52 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2781473) INFO 07-12 17:22:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2781293) WARNING 07-12 17:22:52 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8709
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2781293) INFO: Started server process [2781293]
|
||||
(APIServer pid=2781293) INFO: Waiting for application startup.
|
||||
(APIServer pid=2781293) INFO: Application startup complete.
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48878 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2781473) WARNING 07-12 17:22:59 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2781473) WARNING 07-12 17:22:59 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:03 [loggers.py:273] Engine 000: Avg prompt throughput: 3723.9 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.0%, Prefix cache hit rate: 3.1%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:42728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:13 [loggers.py:273] Engine 000: Avg prompt throughput: 1237.3 tokens/s, Avg generation throughput: 112.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:44796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:23 [loggers.py:273] Engine 000: Avg prompt throughput: 741.6 tokens/s, Avg generation throughput: 264.7 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 48.9%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:33 [loggers.py:273] Engine 000: Avg prompt throughput: 6168.6 tokens/s, Avg generation throughput: 349.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 33.8%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:48500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:43 [loggers.py:273] Engine 000: Avg prompt throughput: 2986.6 tokens/s, Avg generation throughput: 154.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 31.8%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:51334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:23:53 [loggers.py:273] Engine 000: Avg prompt throughput: 6698.0 tokens/s, Avg generation throughput: 274.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 25.6%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:50920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:03 [loggers.py:273] Engine 000: Avg prompt throughput: 8748.6 tokens/s, Avg generation throughput: 383.1 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.6%, Prefix cache hit rate: 22.2%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:13 [loggers.py:273] Engine 000: Avg prompt throughput: 7763.4 tokens/s, Avg generation throughput: 314.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 19.4%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:39632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:23 [loggers.py:273] Engine 000: Avg prompt throughput: 2772.7 tokens/s, Avg generation throughput: 66.1 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 18.4%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:55584 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5746.0 tokens/s, Avg generation throughput: 325.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.4%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:38066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:43 [loggers.py:273] Engine 000: Avg prompt throughput: 5414.5 tokens/s, Avg generation throughput: 327.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 16.9%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:36984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:24:53 [loggers.py:273] Engine 000: Avg prompt throughput: 4879.3 tokens/s, Avg generation throughput: 147.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.5%, Prefix cache hit rate: 16.4%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32854 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:32932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:03 [loggers.py:273] Engine 000: Avg prompt throughput: 6254.7 tokens/s, Avg generation throughput: 284.3 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 15.6%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:54354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:13 [loggers.py:273] Engine 000: Avg prompt throughput: 8412.0 tokens/s, Avg generation throughput: 386.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 14.9%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:53884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:23 [loggers.py:273] Engine 000: Avg prompt throughput: 7810.9 tokens/s, Avg generation throughput: 352.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 14.3%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:57882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:33 [loggers.py:273] Engine 000: Avg prompt throughput: 2806.5 tokens/s, Avg generation throughput: 102.5 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.6%, Prefix cache hit rate: 14.0%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60648 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6210.2 tokens/s, Avg generation throughput: 300.2 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 13.7%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:43302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:25:53 [loggers.py:273] Engine 000: Avg prompt throughput: 4335.6 tokens/s, Avg generation throughput: 281.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.6%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33538 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:33616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:03 [loggers.py:273] Engine 000: Avg prompt throughput: 4969.8 tokens/s, Avg generation throughput: 215.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 13.5%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:56240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:13 [loggers.py:273] Engine 000: Avg prompt throughput: 8656.3 tokens/s, Avg generation throughput: 225.0 tokens/s, Running: 15 reqs, Waiting: 1 reqs, GPU KV cache usage: 16.7%, Prefix cache hit rate: 13.1%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:40990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:41000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:41016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:23 [loggers.py:273] Engine 000: Avg prompt throughput: 7267.6 tokens/s, Avg generation throughput: 393.8 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.3%, Prefix cache hit rate: 12.8%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:49326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5850.9 tokens/s, Avg generation throughput: 318.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.6%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:58518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6442.0 tokens/s, Avg generation throughput: 165.8 tokens/s, Running: 15 reqs, Waiting: 2 reqs, GPU KV cache usage: 15.1%, Prefix cache hit rate: 12.2%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:59998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:60332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO 07-12 17:26:53 [loggers.py:273] Engine 000: Avg prompt throughput: 9543.6 tokens/s, Avg generation throughput: 465.2 tokens/s, Running: 15 reqs, Waiting: 19 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 12.1%
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2781293) INFO: 127.0.0.1:37780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.0858641,
|
||||
"end_mono_ns": 208560052503917,
|
||||
"end_wall_ns": 1783876987178930964,
|
||||
"start_mono_ns": 208551966639817,
|
||||
"start_wall_ns": 1783876979093067067
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.565764511746583,
|
||||
"min": 5.336474936856983,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 692.4971300177276,
|
||||
"min": 73.91095400089398,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.276838561,
|
||||
"end_mono_ns": 208934443492979,
|
||||
"end_wall_ns": 1783877361569920155,
|
||||
"start_mono_ns": 208873166654418,
|
||||
"start_wall_ns": 1783877300293081595
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 35.652001740137756,
|
||||
"min": 4.437448913378157,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1483.3475630148314,
|
||||
"min": 36.238638014765456,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.286066924,
|
||||
"end_mono_ns": 209002647122468,
|
||||
"end_wall_ns": 1783877429773549925,
|
||||
"start_mono_ns": 208941361055544,
|
||||
"start_wall_ns": 1783877368487482719
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 45.35468105514585,
|
||||
"min": 4.485542401506412,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1783.296645997325,
|
||||
"min": 46.25170599319972,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.483328476,
|
||||
"end_mono_ns": 209138933842058,
|
||||
"end_wall_ns": 1783877566060269438,
|
||||
"start_mono_ns": 209077450513582,
|
||||
"start_wall_ns": 1783877504576941248
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 47.06143370878105,
|
||||
"min": 4.492237472331049,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1786.8237869988661,
|
||||
"min": 48.965324996970594,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.09057971014492754, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 53,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 14.566282774,
|
||||
"end_mono_ns": 209227196674186,
|
||||
"end_wall_ns": 1783877654323101400,
|
||||
"start_mono_ns": 209212630391412,
|
||||
"start_wall_ns": 1783877639756818908
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.09057971014492754,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 4.6,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 25,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 53,
|
||||
"finite_n": 53,
|
||||
"max": 64.01803995268126,
|
||||
"min": 15.782681448700448,
|
||||
"missing_n": 223,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 53,
|
||||
"finite_n": 53,
|
||||
"max": 1837.5349910056684,
|
||||
"min": 66.4229110116139,
|
||||
"missing_n": 223,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp1_mns32",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 22265,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2786356-1783877278912197439.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8710 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/anchor-0.2421875
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.24609375 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/anchor-0.24609375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.24609375 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/confirm-1-anchor-0.24609375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.25 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.25 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/confirm-1-anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.5 --tp 1 --mns 32 --base-url http://127.0.0.1:8710 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.273475712,
|
||||
"end_mono_ns": 209070710390123,
|
||||
"end_wall_ns": 1783877497836817221,
|
||||
"start_mono_ns": 209009436914411,
|
||||
"start_wall_ns": 1783877436563341828
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 43.52338799214959,
|
||||
"min": 4.487348055168179,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1779.3545550084673,
|
||||
"min": 48.642217007,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.498563674,
|
||||
"end_mono_ns": 209206240674999,
|
||||
"end_wall_ns": 1783877633367101961,
|
||||
"start_mono_ns": 209144742111325,
|
||||
"start_wall_ns": 1783877571868538807
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 45.34078648040717,
|
||||
"min": 4.47612570064311,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1783.0426309956238,
|
||||
"min": 52.536423987476155,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2786356-1783877278912197439.jsonl","encoded_records":22265,"written_records":22265,"dropped_records":0,"last_step_index":22264,"checkpoint_wall_ns":1783877655001716337,"flush_interval_seconds":1.0,"final":true}
|
||||
@@ -0,0 +1,937 @@
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339]
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:339]
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8710, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 32, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:18 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:28 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:30 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:56647 backend=nccl
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:30 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:31 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:31 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:31 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:31 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:31 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:32 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1288.53 GiB.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:32 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:12, 1.17it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:12, 1.12it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:13, 1.01s/it]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:03<00:11, 1.03it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:04<00:10, 1.06it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:05<00:09, 1.07it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:06<00:08, 1.08it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:07<00:07, 1.10it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:08<00:06, 1.12it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:09<00:05, 1.13it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:10<00:04, 1.14it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:10<00:03, 1.13it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:11<00:02, 1.13it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:12<00:01, 1.13it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:13<00:00, 1.12it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.43it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.16it/s]
|
||||
(EngineCore pid=2786356)
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:46 [default_loader.py:430] Loading weights took 13.86 seconds
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:46 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:46 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 14.484264 seconds
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:49 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/30ef41b5f5/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:49 [backends.py:1148] Dynamo bytecode transform time: 3.07 s
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:52 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.227 s
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:52 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/738c624149a63d13eaf115eec4d2189ece948ac500e524d3e06e801d9915352d/rank_0_0/model
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:52 [monitor.py:53] torch.compile took 5.68 s in total
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:52 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:52 [monitor.py:81] Initial profiling/warmup run took 0.25 s
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:53 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=11 (largest=64), FULL=7 (largest=32)
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.11 GiB total
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [gpu_worker.py:508] Available KV cache memory: 29.4 GiB
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9188 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9212. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [kv_cache_utils.py:2146] GPU KV cache size: 321,136 tokens
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.84x
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:55 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2786356) 2026-07-12 17:27:55,549 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2786356) 2026-07-12 17:27:55,596 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2786356)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:00<00:00, 10.41it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:00<00:00, 10.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:00<00:00, 10.45it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:00<00:00, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:01<00:00, 8.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:01<00:00, 8.02it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 7.68it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 8.52it/s]
|
||||
(EngineCore pid=2786356)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 2/7 [00:00<00:00, 11.62it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 4/7 [00:00<00:00, 11.67it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 6/7 [00:00<00:00, 11.85it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 11.84it/s]
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.13 GiB
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [gpu_worker.py:667] CUDA graph pool memory: 0.13 GiB (actual), 0.11 GiB (estimated), difference: 0.02 GiB (12.1%).
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.67 s (compilation: 5.68 s)
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2786356-1783877278912197439.jsonl
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2786356) INFO 07-12 17:27:58 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:58 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2786173) WARNING 07-12 17:27:59 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8710
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2786173) INFO 07-12 17:27:59 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2786173) INFO: Started server process [2786173]
|
||||
(APIServer pid=2786173) INFO: Waiting for application startup.
|
||||
(APIServer pid=2786173) INFO: Application startup complete.
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38604 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2786356) WARNING 07-12 17:28:06 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2786356) WARNING 07-12 17:28:07 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:38682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:09 [loggers.py:273] Engine 000: Avg prompt throughput: 3320.8 tokens/s, Avg generation throughput: 65.0 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.9%, Prefix cache hit rate: 3.1%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:46870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:46874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:46888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:46890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:46892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:19 [loggers.py:273] Engine 000: Avg prompt throughput: 1678.4 tokens/s, Avg generation throughput: 133.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40476 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:29 [loggers.py:273] Engine 000: Avg prompt throughput: 741.6 tokens/s, Avg generation throughput: 274.4 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 45.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:39 [loggers.py:273] Engine 000: Avg prompt throughput: 6780.8 tokens/s, Avg generation throughput: 349.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 33.8%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59754 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:49 [loggers.py:273] Engine 000: Avg prompt throughput: 2089.4 tokens/s, Avg generation throughput: 131.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 31.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:28:59 [loggers.py:273] Engine 000: Avg prompt throughput: 6406.3 tokens/s, Avg generation throughput: 279.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 26.5%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:09 [loggers.py:273] Engine 000: Avg prompt throughput: 7668.2 tokens/s, Avg generation throughput: 381.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 23.1%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55584 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:19 [loggers.py:273] Engine 000: Avg prompt throughput: 7347.0 tokens/s, Avg generation throughput: 290.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 20.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:55798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:29 [loggers.py:273] Engine 000: Avg prompt throughput: 2695.1 tokens/s, Avg generation throughput: 49.5 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 19.1%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5746.5 tokens/s, Avg generation throughput: 324.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58584 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:49 [loggers.py:273] Engine 000: Avg prompt throughput: 5414.2 tokens/s, Avg generation throughput: 327.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 17.4%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:33992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:29:59 [loggers.py:273] Engine 000: Avg prompt throughput: 4879.7 tokens/s, Avg generation throughput: 148.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.5%, Prefix cache hit rate: 16.8%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:09 [loggers.py:273] Engine 000: Avg prompt throughput: 6255.2 tokens/s, Avg generation throughput: 283.8 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 15.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42754 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:19 [loggers.py:273] Engine 000: Avg prompt throughput: 8412.7 tokens/s, Avg generation throughput: 386.2 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 15.2%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58950 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:58998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:29 [loggers.py:273] Engine 000: Avg prompt throughput: 7168.7 tokens/s, Avg generation throughput: 332.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.6%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:39 [loggers.py:273] Engine 000: Avg prompt throughput: 2806.7 tokens/s, Avg generation throughput: 96.2 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.6%, Prefix cache hit rate: 14.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:59998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50476 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:49 [loggers.py:273] Engine 000: Avg prompt throughput: 6210.5 tokens/s, Avg generation throughput: 292.0 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 14.0%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:30:59 [loggers.py:273] Engine 000: Avg prompt throughput: 4336.0 tokens/s, Avg generation throughput: 290.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:42096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:09 [loggers.py:273] Engine 000: Avg prompt throughput: 4970.2 tokens/s, Avg generation throughput: 207.8 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 13.8%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36876 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:36888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:19 [loggers.py:273] Engine 000: Avg prompt throughput: 8751.6 tokens/s, Avg generation throughput: 213.8 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 18.4%, Prefix cache hit rate: 13.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:60412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6532.7 tokens/s, Avg generation throughput: 447.7 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.0%, Prefix cache hit rate: 13.0%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:53120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.1 tokens/s, Avg generation throughput: 257.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.8%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:49 [loggers.py:273] Engine 000: Avg prompt throughput: 4474.4 tokens/s, Avg generation throughput: 147.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 12.6%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34538 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34648 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:31:59 [loggers.py:273] Engine 000: Avg prompt throughput: 6552.3 tokens/s, Avg generation throughput: 293.1 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.1%, Prefix cache hit rate: 12.5%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:49994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:09 [loggers.py:273] Engine 000: Avg prompt throughput: 3238.5 tokens/s, Avg generation throughput: 268.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 12.5%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:50056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41950 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:41956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4692.5 tokens/s, Avg generation throughput: 263.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.4%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:29 [loggers.py:273] Engine 000: Avg prompt throughput: 8338.4 tokens/s, Avg generation throughput: 299.1 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.6%, Prefix cache hit rate: 12.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:47990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39876 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8498.7 tokens/s, Avg generation throughput: 312.5 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.1%, Prefix cache hit rate: 11.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:40086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:34710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:49 [loggers.py:273] Engine 000: Avg prompt throughput: 4302.5 tokens/s, Avg generation throughput: 246.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.9%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:32:59 [loggers.py:273] Engine 000: Avg prompt throughput: 5531.4 tokens/s, Avg generation throughput: 197.4 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:44102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:09 [loggers.py:273] Engine 000: Avg prompt throughput: 6254.0 tokens/s, Avg generation throughput: 387.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:19 [loggers.py:273] Engine 000: Avg prompt throughput: 2883.5 tokens/s, Avg generation throughput: 168.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6387.2 tokens/s, Avg generation throughput: 257.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, Prefix cache hit rate: 11.6%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:39236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:39 [loggers.py:273] Engine 000: Avg prompt throughput: 9079.1 tokens/s, Avg generation throughput: 346.6 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.4%, Prefix cache hit rate: 11.6%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:56502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8292.8 tokens/s, Avg generation throughput: 338.9 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.7%, Prefix cache hit rate: 11.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:48616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO 07-12 17:33:59 [loggers.py:273] Engine 000: Avg prompt throughput: 1669.9 tokens/s, Avg generation throughput: 133.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.3%
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:37940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2786173) INFO: 127.0.0.1:57554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.025560927,
|
||||
"end_mono_ns": 208867087945013,
|
||||
"end_wall_ns": 1783877294214371984,
|
||||
"start_mono_ns": 208859062384086,
|
||||
"start_wall_ns": 1783877286188811199
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 32,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 29.323968622012167,
|
||||
"min": 4.965293204675861,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1018.7045119819231,
|
||||
"min": 78.0059949902352,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.278139352,
|
||||
"end_mono_ns": 209486742535416,
|
||||
"end_wall_ns": 1783877913868962814,
|
||||
"start_mono_ns": 209425464396064,
|
||||
"start_wall_ns": 1783877852590823366
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 34.97855922053238,
|
||||
"min": 4.792881425099785,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1473.6714000173379,
|
||||
"min": 39.49895699042827,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.282007271,
|
||||
"end_mono_ns": 209554942853886,
|
||||
"end_wall_ns": 1783877982069281476,
|
||||
"start_mono_ns": 209493660846615,
|
||||
"start_wall_ns": 1783877920787273687
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 48.099267582718916,
|
||||
"min": 4.4906551889638875,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1780.4366719792597,
|
||||
"min": 50.33687801915221,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.473047427,
|
||||
"end_mono_ns": 209690427848013,
|
||||
"end_wall_ns": 1783878117554274971,
|
||||
"start_mono_ns": 209628954800586,
|
||||
"start_wall_ns": 1783878056081227964
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 48.026050653631295,
|
||||
"min": 4.490948598428563,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1785.189588990761,
|
||||
"min": 47.32625198084861,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.06159420289855073, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 80,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 20.200314331,
|
||||
"end_mono_ns": 209785367162483,
|
||||
"end_wall_ns": 1783878212493589584,
|
||||
"start_mono_ns": 209765166848152,
|
||||
"start_wall_ns": 1783878192293275756
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.06159420289855073,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 4.6,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 17,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 80,
|
||||
"finite_n": 80,
|
||||
"max": 105.92603693692395,
|
||||
"min": 21.502651858343338,
|
||||
"missing_n": 196,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 80,
|
||||
"finite_n": 80,
|
||||
"max": 1093.1891240179539,
|
||||
"min": 69.8842880083248,
|
||||
"missing_n": 196,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp1_mns64",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 22050,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2795014-1783877830485564578.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8711 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/anchor-0.2421875
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.24609375 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/anchor-0.24609375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.24609375 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/confirm-1-anchor-0.24609375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.25 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.25 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/confirm-1-anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.5 --tp 1 --mns 64 --base-url http://127.0.0.1:8711 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.282251823,
|
||||
"end_mono_ns": 209622963827674,
|
||||
"end_wall_ns": 1783878050090255139,
|
||||
"start_mono_ns": 209561681575851,
|
||||
"start_wall_ns": 1783877988808003321
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 48.01008399226703,
|
||||
"min": 4.492254126138662,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1784.66858502361,
|
||||
"min": 49.72933098906651,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.478525423,
|
||||
"end_mono_ns": 209758399063288,
|
||||
"end_wall_ns": 1783878185525489953,
|
||||
"start_mono_ns": 209696920537865,
|
||||
"start_wall_ns": 1783878124046965275
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 48.10000394492073,
|
||||
"min": 4.492832637643148,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1776.7020479950588,
|
||||
"min": 48.44904300989583,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2795014-1783877830485564578.jsonl","encoded_records":22050,"written_records":22050,"dropped_records":0,"last_step_index":22049,"checkpoint_wall_ns":1783878213124549218,"flush_interval_seconds":1.0,"final":true}
|
||||
@@ -0,0 +1,965 @@
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339]
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:339]
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8711, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2794784) INFO 07-12 17:36:28 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:37 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:40 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:53983 backend=nccl
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:40 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1288.62 GiB.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:41 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:12, 1.17it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:12, 1.13it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:11, 1.13it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:03<00:10, 1.11it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:04<00:09, 1.11it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:05<00:09, 1.09it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:06<00:08, 1.09it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:07<00:07, 1.11it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:08<00:06, 1.12it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:09<00:05, 1.10it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:09<00:04, 1.09it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:10<00:03, 1.11it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:11<00:02, 1.10it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:12<00:01, 1.10it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:13<00:00, 1.10it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.40it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.16it/s]
|
||||
(EngineCore pid=2795014)
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:55 [default_loader.py:430] Loading weights took 13.83 seconds
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:55 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:56 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 14.487128 seconds
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:59 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/65b50dadd2/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2795014) INFO 07-12 17:36:59 [backends.py:1148] Dynamo bytecode transform time: 3.25 s
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:02 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.195 s
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:02 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/ab53f49ec98f407fe24fecb037cb59739264f283939c70f41986d8369686d472/rank_0_0/model
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:02 [monitor.py:53] torch.compile took 5.84 s in total
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:02 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:02 [monitor.py:81] Initial profiling/warmup run took 0.21 s
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:03 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:04 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.19 GiB total
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [gpu_worker.py:508] Available KV cache memory: 29.33 GiB
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9180 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9220. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [kv_cache_utils.py:2146] GPU KV cache size: 320,304 tokens
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.82x
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:05 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2795014) 2026-07-12 17:37:05,528 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2795014) 2026-07-12 17:37:05,599 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2795014)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:01, 9.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:01, 9.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:01, 9.58it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:00<00:01, 9.37it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:01, 9.40it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:00<00:01, 9.32it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:00<00:01, 9.13it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:01, 8.96it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:00<00:01, 9.13it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:01<00:00, 9.25it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:01<00:00, 9.26it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:01<00:00, 8.24it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:01<00:00, 8.56it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:01<00:00, 7.84it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:01<00:00, 7.48it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:01<00:00, 7.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:02<00:00, 7.05it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:02<00:00, 7.06it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 6.90it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 8.12it/s]
|
||||
(EngineCore pid=2795014)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 10.54it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 10.74it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 10.88it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 10.76it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:00<00:00, 10.96it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:01<00:00, 10.93it/s]
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:09 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.24 GiB
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:09 [gpu_worker.py:667] CUDA graph pool memory: 0.24 GiB (actual), 0.19 GiB (estimated), difference: 0.05 GiB (19.8%).
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:09 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:09 [core.py:337] init engine (profile, create kv cache, warmup model) took 13.56 s (compilation: 5.84 s)
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:10 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2795014-1783877830485564578.jsonl
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:10 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2795014) INFO 07-12 17:37:10 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:10 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2794784) WARNING 07-12 17:37:10 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8711
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:11 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2794784) INFO: Started server process [2794784]
|
||||
(APIServer pid=2794784) INFO: Waiting for application startup.
|
||||
(APIServer pid=2794784) INFO: Application startup complete.
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58048 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2795014) WARNING 07-12 17:37:18 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2795014) WARNING 07-12 17:37:18 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:21 [loggers.py:273] Engine 000: Avg prompt throughput: 2652.2 tokens/s, Avg generation throughput: 72.9 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.6%, Prefix cache hit rate: 3.3%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:31 [loggers.py:273] Engine 000: Avg prompt throughput: 2402.9 tokens/s, Avg generation throughput: 125.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38854 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:41 [loggers.py:273] Engine 000: Avg prompt throughput: 741.6 tokens/s, Avg generation throughput: 249.6 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 48.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:37:51 [loggers.py:273] Engine 000: Avg prompt throughput: 5886.8 tokens/s, Avg generation throughput: 361.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 34.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:01 [loggers.py:273] Engine 000: Avg prompt throughput: 2626.0 tokens/s, Avg generation throughput: 142.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 32.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:11 [loggers.py:273] Engine 000: Avg prompt throughput: 6273.9 tokens/s, Avg generation throughput: 257.9 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 27.0%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49854 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49876 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:21 [loggers.py:273] Engine 000: Avg prompt throughput: 8155.7 tokens/s, Avg generation throughput: 367.8 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.6%, Prefix cache hit rate: 23.1%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:31 [loggers.py:273] Engine 000: Avg prompt throughput: 7168.1 tokens/s, Avg generation throughput: 301.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 20.2%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:41 [loggers.py:273] Engine 000: Avg prompt throughput: 2241.2 tokens/s, Avg generation throughput: 73.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 19.5%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:43926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:38:51 [loggers.py:273] Engine 000: Avg prompt throughput: 5967.3 tokens/s, Avg generation throughput: 305.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 18.1%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:01 [loggers.py:273] Engine 000: Avg prompt throughput: 5765.1 tokens/s, Avg generation throughput: 333.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.2%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:36874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:11 [loggers.py:273] Engine 000: Avg prompt throughput: 4274.7 tokens/s, Avg generation throughput: 133.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.5%, Prefix cache hit rate: 16.8%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:56520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48538 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:21 [loggers.py:273] Engine 000: Avg prompt throughput: 6133.1 tokens/s, Avg generation throughput: 312.9 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.4%, Prefix cache hit rate: 16.1%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:48738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:31 [loggers.py:273] Engine 000: Avg prompt throughput: 8234.5 tokens/s, Avg generation throughput: 372.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 15.3%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58854 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:58858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:41 [loggers.py:273] Engine 000: Avg prompt throughput: 8134.1 tokens/s, Avg generation throughput: 319.7 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 14.6%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:39:51 [loggers.py:273] Engine 000: Avg prompt throughput: 2450.0 tokens/s, Avg generation throughput: 106.6 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hit rate: 14.4%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:01 [loggers.py:273] Engine 000: Avg prompt throughput: 6478.6 tokens/s, Avg generation throughput: 257.9 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 13.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:60816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:11 [loggers.py:273] Engine 000: Avg prompt throughput: 4424.8 tokens/s, Avg generation throughput: 339.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:21 [loggers.py:273] Engine 000: Avg prompt throughput: 4367.7 tokens/s, Avg generation throughput: 185.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 13.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:31 [loggers.py:273] Engine 000: Avg prompt throughput: 7697.0 tokens/s, Avg generation throughput: 225.4 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 16.7%, Prefix cache hit rate: 13.2%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:53984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:41 [loggers.py:273] Engine 000: Avg prompt throughput: 7694.6 tokens/s, Avg generation throughput: 448.6 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.3%, Prefix cache hit rate: 13.1%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47642 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:40:51 [loggers.py:273] Engine 000: Avg prompt throughput: 6341.9 tokens/s, Avg generation throughput: 267.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.8%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:01 [loggers.py:273] Engine 000: Avg prompt throughput: 4474.1 tokens/s, Avg generation throughput: 150.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 12.6%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:51230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:11 [loggers.py:273] Engine 000: Avg prompt throughput: 6553.7 tokens/s, Avg generation throughput: 298.5 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.7%, Prefix cache hit rate: 12.5%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:21 [loggers.py:273] Engine 000: Avg prompt throughput: 3276.6 tokens/s, Avg generation throughput: 261.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 12.5%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:52586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:31 [loggers.py:273] Engine 000: Avg prompt throughput: 4826.0 tokens/s, Avg generation throughput: 261.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 12.5%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47560 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:47608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:41 [loggers.py:273] Engine 000: Avg prompt throughput: 8166.5 tokens/s, Avg generation throughput: 296.0 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.7%, Prefix cache hit rate: 12.3%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:45560 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:41:51 [loggers.py:273] Engine 000: Avg prompt throughput: 8498.4 tokens/s, Avg generation throughput: 315.9 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.2%, Prefix cache hit rate: 11.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:38994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:39092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:01 [loggers.py:273] Engine 000: Avg prompt throughput: 4302.3 tokens/s, Avg generation throughput: 245.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.9%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:11 [loggers.py:273] Engine 000: Avg prompt throughput: 4518.7 tokens/s, Avg generation throughput: 191.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33648 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:33664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:21 [loggers.py:273] Engine 000: Avg prompt throughput: 7035.5 tokens/s, Avg generation throughput: 369.9 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57754 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:57834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:31 [loggers.py:273] Engine 000: Avg prompt throughput: 3034.9 tokens/s, Avg generation throughput: 181.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:59420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:41 [loggers.py:273] Engine 000: Avg prompt throughput: 5391.6 tokens/s, Avg generation throughput: 257.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 11.7%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54950 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:54998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:55048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:42:51 [loggers.py:273] Engine 000: Avg prompt throughput: 10077.3 tokens/s, Avg generation throughput: 322.3 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.1%, Prefix cache hit rate: 11.6%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:50540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:43:01 [loggers.py:273] Engine 000: Avg prompt throughput: 8366.7 tokens/s, Avg generation throughput: 335.8 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.2%, Prefix cache hit rate: 11.3%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:44438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:43:11 [loggers.py:273] Engine 000: Avg prompt throughput: 1671.8 tokens/s, Avg generation throughput: 172.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.3%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:35348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO 07-12 17:43:21 [loggers.py:273] Engine 000: Avg prompt throughput: 11103.3 tokens/s, Avg generation throughput: 251.9 tokens/s, Running: 33 reqs, Waiting: 0 reqs, GPU KV cache usage: 27.5%, Prefix cache hit rate: 11.1%
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2794784) INFO: 127.0.0.1:49274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.077432828,
|
||||
"end_mono_ns": 209418942262065,
|
||||
"end_wall_ns": 1783877846068689190,
|
||||
"start_mono_ns": 209410864829237,
|
||||
"start_wall_ns": 1783877837991256103
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 64,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.741828976272362,
|
||||
"min": 5.349175684845192,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 682.0790720230434,
|
||||
"min": 56.28180899657309,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.21875, "kind": "anchor", "pass_rate": 0.9917355371900827, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.21875,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 121,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.272714175,
|
||||
"end_mono_ns": 208049949471616,
|
||||
"end_wall_ns": 1783876477075898873,
|
||||
"start_mono_ns": 207988676757441,
|
||||
"start_wall_ns": 1783876415803184668
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 121,
|
||||
"pass_rate": 0.9917355371900827,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "f585e4793a8a6a621dbfe622514743ac316e34e3e93aaa1c6ba5f1eb58e8ff2d",
|
||||
"arrival_s": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"count": 121,
|
||||
"long_gt4096": 42,
|
||||
"offered_req_s": 2.0166666666666666,
|
||||
"offered_req_s_per_gpu": 2.0166666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 114,
|
||||
"finite_n": 121,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"raw_length_order_sha256": "061592e591871368954f4f80ee75cde2cfc399bd719f3f5deaad275e9af72e8b",
|
||||
"request_id_order_sha256": "9b8c177ea00e5e42bd3294328201cb57addfd823913a5721714ce69646d715f4"
|
||||
},
|
||||
"slo_pass_count": 120,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 30.270937291215958,
|
||||
"min": 4.492692204786274,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 2006.5461480116937,
|
||||
"min": 70.57557700318284,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "anchor", "pass_rate": 0.9920634920634921, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 126,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.279043859,
|
||||
"end_mono_ns": 207914196717219,
|
||||
"end_wall_ns": 1783876341323144094,
|
||||
"start_mono_ns": 207852917673360,
|
||||
"start_wall_ns": 1783876280044100637
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 126,
|
||||
"pass_rate": 0.9920634920634921,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1b67241583bd6f75b1ad3f6d8c86bf0c815306a28f18ea49b1511fed6c091abe",
|
||||
"arrival_s": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"count": 126,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1,
|
||||
"offered_req_s_per_gpu": 2.1,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 119,
|
||||
"finite_n": 126,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"raw_length_order_sha256": "29e3240c20df0cc68172d39aeae84d79908acf564c1b1e2edd447a984e7bdb0e",
|
||||
"request_id_order_sha256": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90"
|
||||
},
|
||||
"slo_pass_count": 125,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 30.22168563792526,
|
||||
"min": 4.490380700884797,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 2057.7456309983972,
|
||||
"min": 41.72074800590053,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.23046875, "kind": "anchor", "pass_rate": 0.9923076923076923, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.23046875,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 130,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.272080003,
|
||||
"end_mono_ns": 208117172178971,
|
||||
"end_wall_ns": 1783876544298605842,
|
||||
"start_mono_ns": 208055900098968,
|
||||
"start_wall_ns": 1783876483026526659
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 130,
|
||||
"pass_rate": 0.9923076923076923,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "57bbe1eb7c1bb474cc7e2b0200b6e92bf84e8c5a21849b9aa6589fcb8ad650d5",
|
||||
"arrival_s": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"count": 130,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1666666666666665,
|
||||
"offered_req_s_per_gpu": 2.1666666666666665,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 122,
|
||||
"finite_n": 130,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"raw_length_order_sha256": "30beac66a67ec987e5f99102936df78950092f1d7eaef89738be547a01fd8954",
|
||||
"request_id_order_sha256": "0f4c973153935c8abcd421eb631900670e31af0dc8a33bdf7e4d3d85e7ed7711"
|
||||
},
|
||||
"slo_pass_count": 129,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 30.301648842475164,
|
||||
"min": 4.491634480380196,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 2005.0652289937716,
|
||||
"min": 68.62633599666879,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.234375, "kind": "anchor", "pass_rate": 0.9924242424242424, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.234375,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 132,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.276628632,
|
||||
"end_mono_ns": 208252399388309,
|
||||
"end_wall_ns": 1783876679525815422,
|
||||
"start_mono_ns": 208191122759677,
|
||||
"start_wall_ns": 1783876618249186880
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 132,
|
||||
"pass_rate": 0.9924242424242424,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "3ade8cc1a55a8a75ea339c66a69fae18f60e5a39097714bc35e4eb728d717aeb",
|
||||
"arrival_s": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"count": 132,
|
||||
"long_gt4096": 45,
|
||||
"offered_req_s": 2.2,
|
||||
"offered_req_s_per_gpu": 2.2,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 124,
|
||||
"finite_n": 132,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"raw_length_order_sha256": "14c37ab0a19d72208c80b7ba7c72fd57614f0763f82e0f04453f0e18cb74bc99",
|
||||
"request_id_order_sha256": "14d2a74fffe25e7d9b3e7490c392eae0dc31bb1f3fe6566c9c0a567543bdd6ac"
|
||||
},
|
||||
"slo_pass_count": 131,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 30.4299091888632,
|
||||
"min": 4.494227968545,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 2019.111283996608,
|
||||
"min": 68.64958599908277,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.25, "kind": "anchor", "pass_rate": 0.958041958041958, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 62.126477495,
|
||||
"end_mono_ns": 208388459138035,
|
||||
"end_wall_ns": 1783876815585565186,
|
||||
"start_mono_ns": 208326332660540,
|
||||
"start_wall_ns": 1783876753459087651
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 0.958041958041958,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 32.858955346627205,
|
||||
"min": 4.492996810975771,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 2534.6092930121813,
|
||||
"min": 69.11832699552178,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.09057971014492754, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 73,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 24.795686401,
|
||||
"end_mono_ns": 208488668302515,
|
||||
"end_wall_ns": 1783876915794729601,
|
||||
"start_mono_ns": 208463872616114,
|
||||
"start_wall_ns": 1783876890999043094
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.09057971014492754,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 4.6,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 25,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 73,
|
||||
"finite_n": 73,
|
||||
"max": 23.976862346469314,
|
||||
"min": 4.817576614193853,
|
||||
"missing_n": 203,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 73,
|
||||
"finite_n": 73,
|
||||
"max": 9597.04709201469,
|
||||
"min": 359.1133150039241,
|
||||
"missing_n": 203,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp1_mns8",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 42704,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2770271-1783876258461072221.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8708 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.2265625
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/confirm-1-anchor-0.2265625
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.21875 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.21875
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.23046875 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.23046875
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.23046875 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/confirm-1-anchor-0.23046875
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.234375 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.234375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.234375 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/confirm-1-anchor-0.234375
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.25 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.25 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/confirm-1-anchor-0.25
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.5 --tp 1 --mns 8 --base-url http://127.0.0.1:8708 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns8/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "anchor", "pass_rate": 0.9920634920634921, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 126,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.283628232,
|
||||
"end_mono_ns": 207982157816433,
|
||||
"end_wall_ns": 1783876409284243533,
|
||||
"start_mono_ns": 207920874188201,
|
||||
"start_wall_ns": 1783876348000615426
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 126,
|
||||
"pass_rate": 0.9920634920634921,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1b67241583bd6f75b1ad3f6d8c86bf0c815306a28f18ea49b1511fed6c091abe",
|
||||
"arrival_s": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"count": 126,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1,
|
||||
"offered_req_s_per_gpu": 2.1,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 119,
|
||||
"finite_n": 126,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"raw_length_order_sha256": "29e3240c20df0cc68172d39aeae84d79908acf564c1b1e2edd447a984e7bdb0e",
|
||||
"request_id_order_sha256": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90"
|
||||
},
|
||||
"slo_pass_count": 125,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 30.423986094391456,
|
||||
"min": 4.49271833865104,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 2015.6794769864064,
|
||||
"min": 71.10144800390117,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.23046875, "kind": "anchor", "pass_rate": 0.9923076923076923, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.23046875,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 130,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.281326625,
|
||||
"end_mono_ns": 208184418966888,
|
||||
"end_wall_ns": 1783876611545393714,
|
||||
"start_mono_ns": 208123137640263,
|
||||
"start_wall_ns": 1783876550264067787
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 130,
|
||||
"pass_rate": 0.9923076923076923,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "57bbe1eb7c1bb474cc7e2b0200b6e92bf84e8c5a21849b9aa6589fcb8ad650d5",
|
||||
"arrival_s": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"count": 130,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1666666666666665,
|
||||
"offered_req_s_per_gpu": 2.1666666666666665,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 122,
|
||||
"finite_n": 130,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"raw_length_order_sha256": "30beac66a67ec987e5f99102936df78950092f1d7eaef89738be547a01fd8954",
|
||||
"request_id_order_sha256": "0f4c973153935c8abcd421eb631900670e31af0dc8a33bdf7e4d3d85e7ed7711"
|
||||
},
|
||||
"slo_pass_count": 129,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 30.304548842328284,
|
||||
"min": 4.494556952744547,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 130,
|
||||
"finite_n": 130,
|
||||
"max": 2015.1741319859866,
|
||||
"min": 67.51175099634565,
|
||||
"missing_n": 0,
|
||||
"n": 130
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.234375, "kind": "anchor", "pass_rate": 0.9924242424242424, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.234375,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 132,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.282402784,
|
||||
"end_mono_ns": 208319682497617,
|
||||
"end_wall_ns": 1783876746808924426,
|
||||
"start_mono_ns": 208258400094833,
|
||||
"start_wall_ns": 1783876685526522242
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 132,
|
||||
"pass_rate": 0.9924242424242424,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "3ade8cc1a55a8a75ea339c66a69fae18f60e5a39097714bc35e4eb728d717aeb",
|
||||
"arrival_s": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"count": 132,
|
||||
"long_gt4096": 45,
|
||||
"offered_req_s": 2.2,
|
||||
"offered_req_s_per_gpu": 2.2,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 124,
|
||||
"finite_n": 132,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"raw_length_order_sha256": "14c37ab0a19d72208c80b7ba7c72fd57614f0763f82e0f04453f0e18cb74bc99",
|
||||
"request_id_order_sha256": "14d2a74fffe25e7d9b3e7490c392eae0dc31bb1f3fe6566c9c0a567543bdd6ac"
|
||||
},
|
||||
"slo_pass_count": 131,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 30.46692667715068,
|
||||
"min": 4.496486598578113,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 132,
|
||||
"finite_n": 132,
|
||||
"max": 2019.6287339786068,
|
||||
"min": 68.34458399680443,
|
||||
"missing_n": 0,
|
||||
"n": 132
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.25, "kind": "anchor", "pass_rate": 0.951048951048951, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 62.091254073,
|
||||
"end_mono_ns": 208457089228587,
|
||||
"end_wall_ns": 1783876884215656001,
|
||||
"start_mono_ns": 208394997974514,
|
||||
"start_wall_ns": 1783876822124401690
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 0.951048951048951,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 136,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 32.83134977150833,
|
||||
"min": 4.493729432991626,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 2528.6180730035994,
|
||||
"min": 70.94443199457601,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2770271-1783876258461072221.jsonl","encoded_records":42704,"written_records":42704,"dropped_records":0,"last_step_index":42703,"checkpoint_wall_ns":1783876916689649928,"flush_interval_seconds":1.0,"final":true}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.581494437,
|
||||
"end_mono_ns": 207846806327487,
|
||||
"end_wall_ns": 1783876273932755595,
|
||||
"start_mono_ns": 207838224833050,
|
||||
"start_wall_ns": 1783876265351260583
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 8,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "899b928ee7acd2abc5bf86f6641591111d09f06ec3dbf35a687067b5dfe9ad57",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7.4525000000000095,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 6,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "c99abf7970483c5386d5c34fad5a1b2664c29b8363b5b6e096338ed8e9773d27",
|
||||
"request_id_order_sha256": "9a7cbf2821c1a7b2a498858815a6f31b0d307f2c8a4900d8be107e6a15858964"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 30.37054844885353,
|
||||
"min": 7.362084622049381,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1017.0694060216192,
|
||||
"min": 107.78111399849877,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.4921875,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 269,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.6037635,
|
||||
"end_mono_ns": 207177385667514,
|
||||
"end_wall_ns": 1783875604512094524,
|
||||
"start_mono_ns": 207116781904014,
|
||||
"start_wall_ns": 1783875543908331340
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 269,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"count": 269,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 4.483333333333333,
|
||||
"offered_req_s_per_gpu": 2.2416666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 250,
|
||||
"finite_n": 269,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
|
||||
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
|
||||
},
|
||||
"slo_pass_count": 269,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 7.384683173259822,
|
||||
"min": 4.122870834681799,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 81.90969700808637,
|
||||
"min": 36.26534101204015,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 273,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.288445601,
|
||||
"end_mono_ns": 207043660468133,
|
||||
"end_wall_ns": 1783875470786895019,
|
||||
"start_mono_ns": 206982372022532,
|
||||
"start_wall_ns": 1783875409498449983
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 273,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 25.645890527525843,
|
||||
"min": 4.867463031422537,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 785.8217659813818,
|
||||
"min": 40.98877400974743,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.5, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 276,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.59665648,
|
||||
"end_mono_ns": 207244584143802,
|
||||
"end_wall_ns": 1783875671710570778,
|
||||
"start_mono_ns": 207183987487322,
|
||||
"start_wall_ns": 1783875611113914502
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 2.3,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 276,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 8.08854837806558,
|
||||
"min": 4.161210228344352,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 341.36167902033776,
|
||||
"min": 35.80997997778468,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 47449,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2755628-1783875389707263062.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
SERVER taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8706 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/warmup
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/anchor-0.49609375
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/confirm-1-anchor-0.49609375
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.4921875 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/anchor-0.4921875
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.5 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/anchor-0.5
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.5 --tp 2 --mns 16 --base-url http://127.0.0.1:8706 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns16/confirm-1-anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 273,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.606811621,
|
||||
"end_mono_ns": 207110687436898,
|
||||
"end_wall_ns": 1783875537813863970,
|
||||
"start_mono_ns": 207050080625277,
|
||||
"start_wall_ns": 1783875477207052621
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 273,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 7.352796283515727,
|
||||
"min": 4.158690614180316,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 76.74413800123148,
|
||||
"min": 35.70285899331793,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.5, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 276,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.597814702,
|
||||
"end_mono_ns": 207311802163383,
|
||||
"end_wall_ns": 1783875738928591380,
|
||||
"start_mono_ns": 207251204348681,
|
||||
"start_wall_ns": 1783875678330776089
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 2.3,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 276,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 7.367673905669,
|
||||
"min": 4.163846921260051,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 76.49152100202627,
|
||||
"min": 36.295153986429796,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2755628-1783875389707263062.jsonl","encoded_records":47449,"written_records":47449,"dropped_records":0,"last_step_index":47448,"checkpoint_wall_ns":1783875743226353253,"flush_interval_seconds":1.0,"final":true}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 4.562253894,
|
||||
"end_mono_ns": 206975612849072,
|
||||
"end_wall_ns": 1783875402739276118,
|
||||
"start_mono_ns": 206971050595178,
|
||||
"start_wall_ns": 1783875398177022539
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3.4938000000000105,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 7,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
|
||||
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 17.359529645732184,
|
||||
"min": 6.608307118230594,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 438.70653098565526,
|
||||
"min": 44.08105098991655,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 391,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.741404276,
|
||||
"end_mono_ns": 205690325731221,
|
||||
"end_wall_ns": 1783874117452158150,
|
||||
"start_mono_ns": 205629584326945,
|
||||
"start_wall_ns": 1783874056710754232
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 391,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 9.36014444877093,
|
||||
"min": 5.401530496017113,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 78.69401399511844,
|
||||
"min": 39.25309798796661,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "anchor", "pass_rate": 0.20558375634517767, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 134,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 24.702770822,
|
||||
"end_mono_ns": 205551739158061,
|
||||
"end_wall_ns": 1783873978865585407,
|
||||
"start_mono_ns": 205527036387239,
|
||||
"start_wall_ns": 1783873954162814682
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 394,
|
||||
"pass_rate": 0.20558375634517767,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "60fc43518e79fd46abe59afde67663a762ca17e817202fa52803040ac9056796",
|
||||
"arrival_s": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"count": 394,
|
||||
"long_gt4096": 142,
|
||||
"offered_req_s": 6.566666666666666,
|
||||
"offered_req_s_per_gpu": 3.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 364,
|
||||
"finite_n": 394,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"raw_length_order_sha256": "4a9cc03e351cce8c558b30732418c878fc3b6d24e64bef6717ce7c87c384384f",
|
||||
"request_id_order_sha256": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd"
|
||||
},
|
||||
"slo_pass_count": 81,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 134,
|
||||
"finite_n": 134,
|
||||
"max": 58.49868067706143,
|
||||
"min": 5.973745897590131,
|
||||
"missing_n": 260,
|
||||
"n": 394
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 134,
|
||||
"finite_n": 134,
|
||||
"max": 4140.989850013284,
|
||||
"min": 41.99217198765837,
|
||||
"missing_n": 260,
|
||||
"n": 394
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns32",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 22996,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/opprof/opprof-v1-dp0-pid2730662-1783873935616837232.jsonl"
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8702 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8702 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/warmup
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8702 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/anchor-0.75390625
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8702 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/confirm-1-anchor-0.75390625
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8702 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/anchor-0.75
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8702 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp2_mns32/confirm-1-anchor-0.75
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 391,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.736454527,
|
||||
"end_mono_ns": 205757115140726,
|
||||
"end_wall_ns": 1783874184241567823,
|
||||
"start_mono_ns": 205696378686199,
|
||||
"start_wall_ns": 1783874123505113386
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 391,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 9.374451102353413,
|
||||
"min": 5.516279039411314,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 77.84111899673007,
|
||||
"min": 37.644838012056425,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user