gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

469
runs/opprof-phase3/phase4/capture-p09/controller-state.json Normal file
View File

@@ -0,0 +1,469 @@
{
"arms": {
"OFF": {
"client_pid": 2487398,
"server_pid": 2486295,
"started_at": 1783856664.5207698,
"status": "complete",
"summary": {
"arm": "OFF",
"bucket_tokens": 237911,
"capture_sizes": "default",
"clean_completed": 1184,
"clean_completed_throughput_rps": 4.933333333333334,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7853007119847462,
"e2e_latency_mean_s": 1.681152764688729,
"e2e_latency_p95_s": 3.8763178953449815,
"gpu_hours": 0.13751606033907995,
"graph_hit_steps": 12252,
"graph_miss_rate": 0.045199501246882795,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 16491,
"model_step_duration_ms": 479367.749483,
"model_steps": 12832,
"padding_fraction": 0.08545632610514016,
"padding_tokens": 20331,
"schema": 1,
"token_efficiency_per_ms": 4.5540506267984115,
"useful_tokens": 2183065
}
},
"ON": {
"client_pid": 2479800,
"server_pid": 2477652,
"started_at": 1783856086.549906,
"status": "complete",
"summary": {
"arm": "ON",
"bucket_tokens": 227714,
"capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"clean_completed": 1190,
"clean_completed_throughput_rps": 4.958333333333333,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7876331160077825,
"e2e_latency_mean_s": 1.612280680370388,
"e2e_latency_p95_s": 3.9930475192406423,
"gpu_hours": 0.15887338949574364,
"graph_hit_steps": 14253,
"graph_miss_rate": 0.0389724226282786,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 17579,
"model_step_duration_ms": 478655.543996,
"model_steps": 14831,
"padding_fraction": 0.035658764941988635,
"padding_tokens": 8120,
"schema": 1,
"token_efficiency_per_ms": 4.562222306607711,
"useful_tokens": 2183733
}
}
},
"controller_pid": 2477456,
"created_at": 1783856081.3481774,
"gpu_hours_increment": 0.2963894498348236,
"plan": {
"added_capture_sizes": [
3,
5,
6,
7,
9
],
"clean_seconds": 240,
"gpu": 0,
"gpu_hour_limit": 16.0,
"load": "moderate",
"manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"measured_padding_recovery_bound": 0.049776380388728225,
"on_capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"order": [
"ON",
"OFF"
],
"pattern": "P09",
"primary_metric": "clean graph-hit padding_fraction",
"prior_gpu_hours": 14.025875418755744,
"profile": false,
"projected_increment_gpu_hours": 0.5,
"projected_total_gpu_hours": 14.525875418755744,
"rate_fraction": 0.6,
"saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
"schema": 1,
"secondary_metrics": [
"Layer-1 useful scheduled tokens/model-step millisecond",
"clean completed request throughput",
"clean request latency"
],
"warmup_seconds": 60
},
"result": {
"delta": {
"completed_throughput_relative": 0.0050675675675675436,
"e2e_mean_latency_relative": -0.04096717785851722,
"e2e_p95_latency_relative": 0.03011353223527924,
"padding_fraction_points": -0.049797561163151524,
"padding_reduction_fraction": 0.5827252753866776,
"token_efficiency_relative": 0.0017943761453185214
},
"gpu_hours_increment": 0.2963894498348236,
"gpu_hours_total": 14.322264868590567,
"off": {
"arm": "OFF",
"bucket_tokens": 237911,
"capture_sizes": "default",
"clean_completed": 1184,
"clean_completed_throughput_rps": 4.933333333333334,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7853007119847462,
"e2e_latency_mean_s": 1.681152764688729,
"e2e_latency_p95_s": 3.8763178953449815,
"gpu_hours": 0.13751606033907995,
"graph_hit_steps": 12252,
"graph_miss_rate": 0.045199501246882795,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 16491,
"model_step_duration_ms": 479367.749483,
"model_steps": 12832,
"padding_fraction": 0.08545632610514016,
"padding_tokens": 20331,
"schema": 1,
"token_efficiency_per_ms": 4.5540506267984115,
"useful_tokens": 2183065
},
"on": {
"arm": "ON",
"bucket_tokens": 227714,
"capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"clean_completed": 1190,
"clean_completed_throughput_rps": 4.958333333333333,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7876331160077825,
"e2e_latency_mean_s": 1.612280680370388,
"e2e_latency_p95_s": 3.9930475192406423,
"gpu_hours": 0.15887338949574364,
"graph_hit_steps": 14253,
"graph_miss_rate": 0.0389724226282786,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 17579,
"model_step_duration_ms": 478655.543996,
"model_steps": 14831,
"padding_fraction": 0.035658764941988635,
"padding_tokens": 8120,
"schema": 1,
"token_efficiency_per_ms": 4.562222306607711,
"useful_tokens": 2183733
},
"plan": {
"added_capture_sizes": [
3,
5,
6,
7,
9
],
"clean_seconds": 240,
"gpu": 0,
"gpu_hour_limit": 16.0,
"load": "moderate",
"manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"measured_padding_recovery_bound": 0.049776380388728225,
"on_capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"order": [
"ON",
"OFF"
],
"pattern": "P09",
"primary_metric": "clean graph-hit padding_fraction",
"prior_gpu_hours": 14.025875418755744,
"profile": false,
"projected_increment_gpu_hours": 0.5,
"projected_total_gpu_hours": 14.525875418755744,
"rate_fraction": 0.6,
"saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
"schema": 1,
"secondary_metrics": [
"Layer-1 useful scheduled tokens/model-step millisecond",
"clean completed request throughput",
"clean request latency"
],
"warmup_seconds": 60
},
"schema": 1
},
"schema": 1,
"status": "complete",
"updated_at": 1783857160.3890784
}

5
runs/opprof-phase3/phase4/capture-p09/controller.log Normal file
View File

@@ -0,0 +1,5 @@
GPU_COMMAND P09-capture-ON-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 --cudagraph-capture-sizes 1 2 3 4 5 6 7 8 9 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512; expected=6-9m
GPU_COMMAND P09-capture-ON-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/on/client; expected=5-7m
GPU_COMMAND P09-capture-OFF-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120; expected=6-9m
GPU_COMMAND P09-capture-OFF-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/off/client; expected=5-7m
{"delta": {"completed_throughput_relative": 0.0050675675675675436, "e2e_mean_latency_relative": -0.04096717785851722, "e2e_p95_latency_relative": 0.03011353223527924, "padding_fraction_points": -0.049797561163151524, "padding_reduction_fraction": 0.5827252753866776, "token_efficiency_relative": 0.0017943761453185214}, "gpu_hours_increment": 0.2963894498348236, "gpu_hours_total": 14.322264868590567, "off": {"arm": "OFF", "bucket_tokens": 237911, "capture_sizes": "default", "clean_completed": 1184, "clean_completed_throughput_rps": 4.933333333333334, "clean_failed": 0, "clean_offered_rps": 4.920833333333333, "drain_seconds": 0.7853007119847462, "e2e_latency_mean_s": 1.681152764688729, "e2e_latency_p95_s": 3.8763178953449815, "gpu_hours": 0.13751606033907995, "graph_hit_steps": 12252, "graph_miss_rate": 0.045199501246882795, "layer1_invariants": {"cudagraph_identity": true, "footer_balanced": true, "footer_written_matches": true, "schema_1": true, "sidecar_agrees": true, "sidecar_final": true, "steps_unique_contiguous": true, "token_composition": true, "zero_drops": true}, "layer1_records": 16491, "model_step_duration_ms": 479367.749483, "model_steps": 12832, "padding_fraction": 0.08545632610514016, "padding_tokens": 20331, "schema": 1, "token_efficiency_per_ms": 4.5540506267984115, "useful_tokens": 2183065}, "on": {"arm": "ON", "bucket_tokens": 227714, "capture_sizes": [1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "clean_completed": 1190, "clean_completed_throughput_rps": 4.958333333333333, "clean_failed": 0, "clean_offered_rps": 4.920833333333333, "drain_seconds": 0.7876331160077825, "e2e_latency_mean_s": 1.612280680370388, "e2e_latency_p95_s": 3.9930475192406423, "gpu_hours": 0.15887338949574364, "graph_hit_steps": 14253, "graph_miss_rate": 0.0389724226282786, "layer1_invariants": {"cudagraph_identity": true, "footer_balanced": true, "footer_written_matches": true, "schema_1": true, "sidecar_agrees": true, "sidecar_final": true, "steps_unique_contiguous": true, "token_composition": true, "zero_drops": true}, "layer1_records": 17579, "model_step_duration_ms": 478655.543996, "model_steps": 14831, "padding_fraction": 0.035658764941988635, "padding_tokens": 8120, "schema": 1, "token_efficiency_per_ms": 4.562222306607711, "useful_tokens": 2183733}, "plan": {"added_capture_sizes": [3, 5, 6, 7, 9], "clean_seconds": 240, "gpu": 0, "gpu_hour_limit": 16.0, "load": "moderate", "manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl", "measured_padding_recovery_bound": 0.049776380388728225, "on_capture_sizes": [1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "order": ["ON", "OFF"], "pattern": "P09", "primary_metric": "clean graph-hit padding_fraction", "prior_gpu_hours": 14.025875418755744, "profile": false, "projected_increment_gpu_hours": 0.5, "projected_total_gpu_hours": 14.525875418755744, "rate_fraction": 0.6, "saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json", "schema": 1, "secondary_metrics": ["Layer-1 useful scheduled tokens/model-step millisecond", "clean completed request throughput", "clean request latency"], "warmup_seconds": 60}, "schema": 1}

1
runs/opprof-phase3/phase4/capture-p09/launch.log Normal file
View File

@@ -0,0 +1 @@
GPU_VALIDATION Phase4 P09 capture sizes: order=ON,OFF, GPU0 TP1, manifest=/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl, saturation_source=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json, added_sizes=3,5,6,7,9, warmup=60s, clean=240s/arm, projected=0.50 H20-hours, cumulative_projection=14.525875/16, expected_wall=15-22m

66
runs/opprof-phase3/phase4/capture-p09/off/client-sanity.json Normal file
View File

@@ -0,0 +1,66 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": true,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"moderate_offered_within_5pct": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 1,
"finite_n": 1477,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1477
},
"admitted_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 299.8483776419889,
"min": 0.00022695399820804596,
"missing_n": 0,
"n": 1477
},
"completed_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 300.78334441498737,
"min": 0.6682249770092312,
"missing_n": 0,
"n": 1477
},
"input_tokens": {
"distinct_n": 944,
"finite_n": 1477,
"max": 8161.0,
"min": 128.0,
"missing_n": 0,
"n": 1477
},
"requested_output_tokens": {
"distinct_n": 1,
"finite_n": 1477,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1477
},
"scheduled_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 299.84763839512016,
"min": 0.0,
"missing_n": 0,
"n": 1477
}
},
"schema": 1
}

2
runs/opprof-phase3/phase4/capture-p09/off/commands.log Normal file
View File

@@ -0,0 +1,2 @@
GPU_COMMAND P09-capture-OFF-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 ; expected=6-9m
GPU_COMMAND P09-capture-OFF-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/off/client ; expected=5-7m

34
runs/opprof-phase3/phase4/capture-p09/off/summary.json Normal file
View File

@@ -0,0 +1,34 @@
{
"arm": "OFF",
"bucket_tokens": 237911,
"capture_sizes": "default",
"clean_completed": 1184,
"clean_completed_throughput_rps": 4.933333333333334,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7853007119847462,
"e2e_latency_mean_s": 1.681152764688729,
"e2e_latency_p95_s": 3.8763178953449815,
"gpu_hours": 0.13751606033907995,
"graph_hit_steps": 12252,
"graph_miss_rate": 0.045199501246882795,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 16491,
"model_step_duration_ms": 479367.749483,
"model_steps": 12832,
"padding_fraction": 0.08545632610514016,
"padding_tokens": 20331,
"schema": 1,
"token_efficiency_per_ms": 4.5540506267984115,
"useful_tokens": 2183065
}

66
runs/opprof-phase3/phase4/capture-p09/on/client-sanity.json Normal file
View File

@@ -0,0 +1,66 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": true,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"moderate_offered_within_5pct": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 1,
"finite_n": 1477,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1477
},
"admitted_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 299.8487557930057,
"min": 0.00024887899053283036,
"missing_n": 0,
"n": 1477
},
"completed_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 300.78616064199014,
"min": 15.648636110010557,
"missing_n": 0,
"n": 1477
},
"input_tokens": {
"distinct_n": 944,
"finite_n": 1477,
"max": 8161.0,
"min": 128.0,
"missing_n": 0,
"n": 1477
},
"requested_output_tokens": {
"distinct_n": 1,
"finite_n": 1477,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1477
},
"scheduled_s": {
"distinct_n": 1477,
"finite_n": 1477,
"max": 299.84763839512016,
"min": 0.0,
"missing_n": 0,
"n": 1477
}
},
"schema": 1
}

2
runs/opprof-phase3/phase4/capture-p09/on/commands.log Normal file
View File

@@ -0,0 +1,2 @@
GPU_COMMAND P09-capture-ON-server: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8200 --tensor-parallel-size 1 --enable-chunked-prefill --enable-prefix-caching --shutdown-timeout 120 --cudagraph-capture-sizes 1 2 3 4 5 6 7 8 9 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512 ; expected=6-9m
GPU_COMMAND P09-capture-ON-client: taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/scripts/opprof_phase3_client.py run --manifest /home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl --base-url http://127.0.0.1:8200 --model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --load-point moderate --saturation-result /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json --rate-fraction 0.60 --max-concurrency 256 --ignore-eos --temperature 0 --warmup-seconds 60 --clean-segment-seconds 80 --num-clean-segments 3 --recovery-seconds 30 --drain-timeout-seconds 120 --workload-seed 20260712 --server-seed 20260712 --result-dir /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase4-capture-p09/on/client ; expected=5-7m

91
runs/opprof-phase3/phase4/capture-p09/on/summary.json Normal file
View File

@@ -0,0 +1,91 @@
{
"arm": "ON",
"bucket_tokens": 227714,
"capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"clean_completed": 1190,
"clean_completed_throughput_rps": 4.958333333333333,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7876331160077825,
"e2e_latency_mean_s": 1.612280680370388,
"e2e_latency_p95_s": 3.9930475192406423,
"gpu_hours": 0.15887338949574364,
"graph_hit_steps": 14253,
"graph_miss_rate": 0.0389724226282786,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 17579,
"model_step_duration_ms": 478655.543996,
"model_steps": 14831,
"padding_fraction": 0.035658764941988635,
"padding_tokens": 8120,
"schema": 1,
"token_efficiency_per_ms": 4.562222306607711,
"useful_tokens": 2183733
}

230
runs/opprof-phase3/phase4/capture-p09/result.json Normal file
View File

@@ -0,0 +1,230 @@
{
"delta": {
"completed_throughput_relative": 0.0050675675675675436,
"e2e_mean_latency_relative": -0.04096717785851722,
"e2e_p95_latency_relative": 0.03011353223527924,
"padding_fraction_points": -0.049797561163151524,
"padding_reduction_fraction": 0.5827252753866776,
"token_efficiency_relative": 0.0017943761453185214
},
"gpu_hours_increment": 0.2963894498348236,
"gpu_hours_total": 14.322264868590567,
"off": {
"arm": "OFF",
"bucket_tokens": 237911,
"capture_sizes": "default",
"clean_completed": 1184,
"clean_completed_throughput_rps": 4.933333333333334,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7853007119847462,
"e2e_latency_mean_s": 1.681152764688729,
"e2e_latency_p95_s": 3.8763178953449815,
"gpu_hours": 0.13751606033907995,
"graph_hit_steps": 12252,
"graph_miss_rate": 0.045199501246882795,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 16491,
"model_step_duration_ms": 479367.749483,
"model_steps": 12832,
"padding_fraction": 0.08545632610514016,
"padding_tokens": 20331,
"schema": 1,
"token_efficiency_per_ms": 4.5540506267984115,
"useful_tokens": 2183065
},
"on": {
"arm": "ON",
"bucket_tokens": 227714,
"capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"clean_completed": 1190,
"clean_completed_throughput_rps": 4.958333333333333,
"clean_failed": 0,
"clean_offered_rps": 4.920833333333333,
"drain_seconds": 0.7876331160077825,
"e2e_latency_mean_s": 1.612280680370388,
"e2e_latency_p95_s": 3.9930475192406423,
"gpu_hours": 0.15887338949574364,
"graph_hit_steps": 14253,
"graph_miss_rate": 0.0389724226282786,
"layer1_invariants": {
"cudagraph_identity": true,
"footer_balanced": true,
"footer_written_matches": true,
"schema_1": true,
"sidecar_agrees": true,
"sidecar_final": true,
"steps_unique_contiguous": true,
"token_composition": true,
"zero_drops": true
},
"layer1_records": 17579,
"model_step_duration_ms": 478655.543996,
"model_steps": 14831,
"padding_fraction": 0.035658764941988635,
"padding_tokens": 8120,
"schema": 1,
"token_efficiency_per_ms": 4.562222306607711,
"useful_tokens": 2183733
},
"plan": {
"added_capture_sizes": [
3,
5,
6,
7,
9
],
"clean_seconds": 240,
"gpu": 0,
"gpu_hour_limit": 16.0,
"load": "moderate",
"manifest": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"measured_padding_recovery_bound": 0.049776380388728225,
"on_capture_sizes": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
16,
24,
32,
40,
48,
56,
64,
72,
80,
88,
96,
104,
112,
120,
128,
136,
144,
152,
160,
168,
176,
184,
192,
200,
208,
216,
224,
232,
240,
248,
256,
272,
288,
304,
320,
336,
352,
368,
384,
400,
416,
432,
448,
464,
480,
496,
512
],
"order": [
"ON",
"OFF"
],
"pattern": "P09",
"primary_metric": "clean graph-hit padding_fraction",
"prior_gpu_hours": 14.025875418755744,
"profile": false,
"projected_increment_gpu_hours": 0.5,
"projected_total_gpu_hours": 14.525875418755744,
"rate_fraction": 0.6,
"saturation_result": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P09-C00/saturation/client/result.json",
"schema": 1,
"secondary_metrics": [
"Layer-1 useful scheduled tokens/model-step millisecond",
"clean completed request throughput",
"clean request latency"
],
"warmup_seconds": 60
},
"schema": 1
}

373
runs/opprof-phase3/phase4/capture_validation.py Normal file
View File

@@ -0,0 +1,373 @@
#!/usr/bin/env python3
"""One-pair P09 CUDAGraph capture-size validation for OpProf Phase 4."""
from __future__ import annotations
import argparse
import json
import math
import os
import shlex
import signal
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
from typing import Any
import numpy as np
import opprof_phase3_controller as common
import opprof_phase3_matrix as matrix
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712")
ROOT = WORKDIR / "runs/phase4-capture-p09"
PHASE3 = WORKDIR / "runs/phase3"
PRIVATE = Path("/home/admin/cpfs/wjh/opprof-phase3-private/manifests")
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
CLIENT = WORKDIR / "scripts/opprof_phase3_client.py"
STATE = ROOT / "controller-state.json"
GPU = 0
PORT = 8200
PRIOR_GPU_HOURS = 14.025875418755744
GPU_HOUR_LIMIT = 16.0
EXPECTED_INCREMENT_HOURS = 0.5
ADDED_CAPTURE_SIZES = (3, 5, 6, 7, 9)
DEFAULT_CAPTURE_SIZES = (
1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104,
112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200,
208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336,
352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512,
)
ON_CAPTURE_SIZES = tuple(sorted(set(DEFAULT_CAPTURE_SIZES + ADDED_CAPTURE_SIZES)))
def plan() -> dict[str, Any]:
return {
"schema": 1,
"pattern": "P09",
"load": "moderate",
"order": ["ON", "OFF"],
"gpu": GPU,
"warmup_seconds": 60,
"clean_seconds": 240,
"profile": False,
"manifest": str(PRIVATE / "P09.jsonl"),
"saturation_result": str(
PHASE3 / "primary/P09-C00/saturation/client/result.json"
),
"rate_fraction": 0.60,
"added_capture_sizes": list(ADDED_CAPTURE_SIZES),
"on_capture_sizes": list(ON_CAPTURE_SIZES),
"measured_padding_recovery_bound": 0.049776380388728225,
"primary_metric": "clean graph-hit padding_fraction",
"secondary_metrics": [
"Layer-1 useful scheduled tokens/model-step millisecond",
"clean completed request throughput",
"clean request latency",
],
"prior_gpu_hours": PRIOR_GPU_HOURS,
"projected_increment_gpu_hours": EXPECTED_INCREMENT_HOURS,
"projected_total_gpu_hours": PRIOR_GPU_HOURS + EXPECTED_INCREMENT_HOURS,
"gpu_hour_limit": GPU_HOUR_LIMIT,
}
def save_state(state: dict[str, Any]) -> None:
state["updated_at"] = time.time()
state["controller_pid"] = os.getpid()
common.atomic_json(STATE, state)
def wait_ready(server: subprocess.Popen[Any]) -> None:
deadline = time.monotonic() + 300
while time.monotonic() < deadline:
if server.poll() is not None:
raise RuntimeError("server exited before readiness")
try:
with urllib.request.urlopen(
f"http://127.0.0.1:{PORT}/health", timeout=1
) as response:
if response.status == 200:
return
except Exception:
pass
time.sleep(1)
raise TimeoutError("server readiness timeout")
def server_command(arm: str) -> list[str]:
command = [
"taskset", "-c", "0-19", str(VENV / "bin/vllm"), "serve", str(MODEL),
"--host", "127.0.0.1", "--port", str(PORT),
"--tensor-parallel-size", "1", "--enable-chunked-prefill",
"--enable-prefix-caching", "--shutdown-timeout", "120",
]
if arm == "ON":
command.extend(("--cudagraph-capture-sizes", *map(str, ON_CAPTURE_SIZES)))
return command
def client_command(run_dir: Path) -> list[str]:
return [
"taskset", "-c", "0-19", str(VENV / "bin/python"), str(CLIENT), "run",
"--manifest", str(PRIVATE / "P09.jsonl"),
"--base-url", f"http://127.0.0.1:{PORT}", "--model", str(MODEL),
"--load-point", "moderate", "--saturation-result",
str(PHASE3 / "primary/P09-C00/saturation/client/result.json"),
"--rate-fraction", "0.60", "--max-concurrency", "256", "--ignore-eos",
"--temperature", "0", "--warmup-seconds", "60",
"--clean-segment-seconds", "80", "--num-clean-segments", "3",
"--recovery-seconds", "30", "--drain-timeout-seconds", "120",
"--workload-seed", "20260712", "--server-seed", "20260712",
"--result-dir", str(run_dir / "client"),
]
def summarize(run_dir: Path, arm: str) -> dict[str, Any]:
result = json.loads((run_dir / "client/result.json").read_text())
requests = [
json.loads(line)
for line in (run_dir / "client/requests.jsonl").read_text().splitlines()
]
t0 = int(result["t0_mono_ns"])
start, end = t0 + int(60e9), t0 + int(300e9)
stream = next((run_dir / "opprof").glob("*.jsonl"))
records = []
for line in stream.read_text().splitlines():
record = json.loads(line)
if (
"step_index" in record
and start <= int(record["submit_mono_ns"]) < end
):
records.append(record)
model = [record for record in records if record["model_executed"]]
hits = [
record
for record in model
if record["cudagraph"]["hit"]
and int(record["cudagraph"]["bucket_tokens"]) > 0
]
pad = sum(int(record["cudagraph"]["padding_tokens"]) for record in hits)
bucket = sum(int(record["cudagraph"]["bucket_tokens"]) for record in hits)
useful = sum(
int(record["prefill_tokens"]) + int(record["decode_tokens"])
for record in records
)
duration_ms = sum(
(int(record["complete_mono_ns"]) - int(record["submit_mono_ns"])) / 1e6
for record in records
)
completed = [
request
for request in requests
if request["success"] and 60 <= float(request["completed_s"]) < 300
]
e2e = np.asarray(
[float(request["completed_s"] - request["admitted_s"]) for request in completed]
)
layer1 = matrix.validate_layer1(run_dir)
return {
"schema": 1,
"arm": arm,
"capture_sizes": list(ON_CAPTURE_SIZES) if arm == "ON" else "default",
"clean_completed": len(completed),
"clean_failed": int(result["clean"]["failed"]),
"clean_completed_throughput_rps": float(
result["clean"]["completed_throughput_rps"]
),
"clean_offered_rps": float(result["clean"]["offered_rps"]),
"e2e_latency_mean_s": float(e2e.mean()),
"e2e_latency_p95_s": float(np.quantile(e2e, 0.95)),
"model_steps": len(model),
"graph_hit_steps": len(hits),
"padding_tokens": pad,
"bucket_tokens": bucket,
"padding_fraction": pad / bucket,
"graph_miss_rate": sum(not record["cudagraph"]["hit"] for record in model)
/ len(model),
"useful_tokens": useful,
"model_step_duration_ms": duration_ms,
"token_efficiency_per_ms": useful / duration_ms,
"layer1_records": layer1["records"],
"layer1_invariants": layer1["invariants"],
"drain_seconds": float(result["drain_seconds"]),
}
def run_arm(state: dict[str, Any], arm: str) -> None:
if state["arms"].get(arm, {}).get("status") == "complete":
return
run_dir = ROOT / arm.lower()
if run_dir.exists():
run_dir.rename(run_dir.with_name(f"{run_dir.name}.interrupted-{int(time.time())}"))
run_dir.mkdir(parents=True)
common.preflight([GPU], run_dir)
server_cmd = server_command(arm)
client_cmd = client_command(run_dir)
commands_path = run_dir / "commands.log"
common.command_log(commands_path, f"P09-capture-{arm}-server", server_cmd, "6-9m")
common.command_log(commands_path, f"P09-capture-{arm}-client", client_cmd, "5-7m")
print(
f"GPU_COMMAND P09-capture-{arm}-server: {shlex.join(server_cmd)}; "
"expected=6-9m",
flush=True,
)
print(
f"GPU_COMMAND P09-capture-{arm}-client: {shlex.join(client_cmd)}; "
"expected=5-7m",
flush=True,
)
environment = os.environ.copy()
environment.update(
{
"CUDA_VISIBLE_DEVICES": str(GPU),
"VLLM_OPPROF_DIR": str(run_dir / "opprof"),
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"PYTHONUNBUFFERED": "1",
}
)
server_log = (run_dir / "server.log").open("ab", buffering=0)
server_started = time.time()
server = subprocess.Popen(
server_cmd,
cwd=SOURCE,
env=environment,
stdout=server_log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
client = None
client_log = None
monitor = None
owned = {server.pid}
state["arms"][arm] = {
"status": "starting",
"server_pid": server.pid,
"started_at": server_started,
}
save_state(state)
failure = None
try:
wait_ready(server)
monitor = common.Monitor(run_dir / "monitor.jsonl", owned)
monitor.start()
client_log = (run_dir / "client.log").open("ab", buffering=0)
client = subprocess.Popen(
client_cmd,
cwd=WORKDIR,
stdout=client_log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
owned.add(client.pid)
state["arms"][arm].update(status="running", client_pid=client.pid)
save_state(state)
deadline = time.monotonic() + 900
while client.poll() is None and time.monotonic() < deadline:
if server.poll() is not None:
raise RuntimeError("server exited during client load")
if monitor.other_apps:
raise RuntimeError(f"other GPU process appeared: {monitor.other_apps}")
time.sleep(2)
if client.poll() is None:
raise TimeoutError("client exceeded 900 seconds")
if client.returncode:
raise RuntimeError(f"client failed with {client.returncode}")
except Exception as error:
failure = error
finally:
if client is not None and client.poll() is None:
try:
os.killpg(client.pid, signal.SIGKILL)
except ProcessLookupError:
pass
common.stop_servers([server])
server_log.close()
if client_log is not None:
client_log.close()
if monitor is not None:
monitor.stop()
common.verify_idle([GPU], run_dir)
gpu_hours = (time.time() - server_started) / 3600
state["gpu_hours_increment"] += gpu_hours
if PRIOR_GPU_HOURS + state["gpu_hours_increment"] >= GPU_HOUR_LIMIT:
failure = failure or RuntimeError("GPU-hour limit reached")
if failure is not None:
state["arms"][arm].update(status="failed", failure=repr(failure))
state["status"] = "failed"
save_state(state)
raise failure
summary = summarize(run_dir, arm)
summary["gpu_hours"] = gpu_hours
common.atomic_json(run_dir / "summary.json", summary)
state["arms"][arm].update(status="complete", summary=summary)
save_state(state)
def run() -> None:
ROOT.mkdir(parents=True, exist_ok=True)
if PRIOR_GPU_HOURS + EXPECTED_INCREMENT_HOURS >= GPU_HOUR_LIMIT:
raise RuntimeError("projected validation exceeds GPU-hour budget")
if STATE.exists():
state = json.loads(STATE.read_text())
else:
state = {
"schema": 1,
"status": "running",
"created_at": time.time(),
"plan": plan(),
"arms": {},
"gpu_hours_increment": 0.0,
}
for arm in ("ON", "OFF"):
run_arm(state, arm)
on, off = state["arms"]["ON"]["summary"], state["arms"]["OFF"]["summary"]
result = {
"schema": 1,
"plan": plan(),
"on": on,
"off": off,
"delta": {
"padding_fraction_points": on["padding_fraction"] - off["padding_fraction"],
"padding_reduction_fraction": 1 - on["padding_fraction"] / off["padding_fraction"],
"token_efficiency_relative": on["token_efficiency_per_ms"]
/ off["token_efficiency_per_ms"]
- 1,
"completed_throughput_relative": on["clean_completed_throughput_rps"]
/ off["clean_completed_throughput_rps"]
- 1,
"e2e_mean_latency_relative": on["e2e_latency_mean_s"]
/ off["e2e_latency_mean_s"]
- 1,
"e2e_p95_latency_relative": on["e2e_latency_p95_s"]
/ off["e2e_latency_p95_s"]
- 1,
},
"gpu_hours_increment": state["gpu_hours_increment"],
"gpu_hours_total": PRIOR_GPU_HOURS + state["gpu_hours_increment"],
}
common.atomic_json(ROOT / "result.json", result)
state["status"] = "complete"
state["result"] = result
save_state(state)
print(json.dumps(result, sort_keys=True))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("command", choices=("plan", "run"))
args = parser.parse_args()
if args.command == "plan":
print(json.dumps(plan(), indent=2, sort_keys=True))
else:
run()
if __name__ == "__main__":
main()