gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Layerwise under load: overlap benefit survives (bg=16)

Browse Source 
mb7 with background decode load (8/instance). Critical-path transfer overhead
stays ~constant ~90ms for layerwise vs 158/239/749ms baseline (up to 7.9x at
32k), prefill not slowed, KV correct. Confirms the overlap holds on busy
instances. DESIGN.md updated with idle-vs-load table + the two blockers
(chunk-safety, concurrent-transfer safety) that the full 1200-req trace needs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-28 16:30:14 +08:00
parent fec50fa45d
commit e77bdcac5a
5 changed files with 387 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
microbench/connector_tax/layerwise/results/mb7_baseline_bg16.json Normal file
View File

@@ -0,0 +1,140 @@
{
"mode": "baseline",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct",
"raw": [
{
"t_prefill_s": 0.5868483350022871,
"t_xfer_s": 0.19584889299949282,
"t_total_s": 0.7827702419999696,
"cached": 8176,
"mode": "baseline",
"size": 8192,
"rep": 0,
"t_prefill_only_s": 0.5920699099988269,
"kv_gib": 0.75,
"correct": true
},
{
"t_prefill_s": 0.5875704979989678,
"t_xfer_s": 0.1554814909977722,
"t_total_s": 0.7431365060001554,
"cached": 8176,
"mode": "baseline",
"size": 8192,
"rep": 1,
"t_prefill_only_s": 0.5814537600017502,
"kv_gib": 0.75,
"correct": true
},
{
"t_prefill_s": 0.5852241569991747,
"t_xfer_s": 0.15129724399957922,
"t_total_s": 0.7365909610016388,
"cached": 8176,
"mode": "baseline",
"size": 8192,
"rep": 2,
"t_prefill_only_s": 0.5846994370003813,
"kv_gib": 0.75,
"correct": true
},
{
"t_prefill_s": 1.498547145001794,
"t_xfer_s": 0.2475714690008317,
"t_total_s": 1.7462187470009667,
"cached": 16368,
"mode": "baseline",
"size": 16384,
"rep": 0,
"t_prefill_only_s": 1.5670790190015396,
"kv_gib": 1.5,
"correct": true
},
{
"t_prefill_s": 1.5025789940009417,
"t_xfer_s": 0.24532966799961287,
"t_total_s": 1.7479741930001182,
"cached": 16368,
"mode": "baseline",
"size": 16384,
"rep": 1,
"t_prefill_only_s": 1.5008903820016712,
"kv_gib": 1.5,
"correct": true
},
{
"t_prefill_s": 1.5021674179988622,
"t_xfer_s": 0.24640760400143336,
"t_total_s": 1.7486415580024186,
"cached": 16368,
"mode": "baseline",
"size": 16384,
"rep": 2,
"t_prefill_only_s": 1.509417139001016,
"kv_gib": 1.5,
"correct": true
},
{
"t_prefill_s": 4.444555983998725,
"t_xfer_s": 0.4227471090016479,
"t_total_s": 4.86737214599998,
"cached": 32752,
"mode": "baseline",
"size": 32768,
"rep": 0,
"t_prefill_only_s": 4.4467717689985875,
"kv_gib": 3.0,
"correct": true
},
{
"t_prefill_s": 4.442135782999685,
"t_xfer_s": 0.7519038230020669,
"t_total_s": 5.194113359000767,
"cached": 32752,
"mode": "baseline",
"size": 32768,
"rep": 1,
"t_prefill_only_s": 4.445541313998547,
"kv_gib": 3.0,
"correct": true
},
{
"t_prefill_s": 4.439772993999213,
"t_xfer_s": 0.7855456319994119,
"t_total_s": 5.225392060998274,
"cached": 32752,
"mode": "baseline",
"size": 32768,
"rep": 2,
"t_prefill_only_s": 4.442906365002273,
"kv_gib": 3.0,
"correct": true
}
],
"summary": [
{
"size": 8192,
"n": 3,
"pf_only_ms": 584.6994370003813,
"total_ms": 743.1365060001554,
"overhead_ms": 158.43706899977406,
"all_correct": true
},
{
"size": 16384,
"n": 3,
"pf_only_ms": 1509.417139001016,
"total_ms": 1747.9741930001182,
"overhead_ms": 238.5570539991022,
"all_correct": true
},
{
"size": 32768,
"n": 3,
"pf_only_ms": 4445.541313998547,
"total_ms": 5194.113359000767,
"overhead_ms": 748.57204500222,
"all_correct": true
}
]
}

140
microbench/connector_tax/layerwise/results/mb7_layerwise_bg16.json Normal file
View File

@@ -0,0 +1,140 @@
{
"mode": "layerwise",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-Coder-30B-A3B-Instruct",
"raw": [
{
"t_A_s": 0.5905098549992545,
"t_B_s": 0.6900827390018094,
"t_total_s": 0.6904724189989793,
"cached": 8176,
"mode": "layerwise",
"size": 8192,
"rep": 0,
"t_prefill_only_s": 0.5852864849985053,
"kv_gib": 0.75,
"correct": true
},
{
"t_A_s": 0.5897548109969648,
"t_B_s": 0.6827381169969158,
"t_total_s": 0.6828304180016858,
"cached": 8176,
"mode": "layerwise",
"size": 8192,
"rep": 1,
"t_prefill_only_s": 0.5890174580017629,
"kv_gib": 0.75,
"correct": true
},
{
"t_A_s": 0.5850713190011447,
"t_B_s": 0.6744917560026806,
"t_total_s": 0.6745770380002796,
"cached": 8176,
"mode": "layerwise",
"size": 8192,
"rep": 2,
"t_prefill_only_s": 0.5943713950000529,
"kv_gib": 0.75,
"correct": true
},
{
"t_A_s": 1.5030149390004226,
"t_B_s": 1.596173029000056,
"t_total_s": 1.597060264000902,
"cached": 16368,
"mode": "layerwise",
"size": 16384,
"rep": 0,
"t_prefill_only_s": 1.5130829510017065,
"kv_gib": 1.5,
"correct": true
},
{
"t_A_s": 1.499876754998695,
"t_B_s": 1.5940461120007967,
"t_total_s": 1.5948001770011615,
"cached": 16368,
"mode": "layerwise",
"size": 16384,
"rep": 1,
"t_prefill_only_s": 1.5024838620010996,
"kv_gib": 1.5,
"correct": true
},
{
"t_A_s": 1.5068977490009274,
"t_B_s": 1.5950395179970656,
"t_total_s": 1.59571184500237,
"cached": 16368,
"mode": "layerwise",
"size": 16384,
"rep": 2,
"t_prefill_only_s": 1.5303227439981129,
"kv_gib": 1.5,
"correct": true
},
{
"t_A_s": 4.4503932609986805,
"t_B_s": 4.538851200999488,
"t_total_s": 4.539281312001549,
"cached": 32752,
"mode": "layerwise",
"size": 32768,
"rep": 0,
"t_prefill_only_s": 4.446753306998289,
"kv_gib": 3.0,
"correct": true
},
{
"t_A_s": 4.44226107799841,
"t_B_s": 4.551636377997056,
"t_total_s": 4.552389411001059,
"cached": 32752,
"mode": "layerwise",
"size": 32768,
"rep": 1,
"t_prefill_only_s": 4.44538704000297,
"kv_gib": 3.0,
"correct": true
},
{
"t_A_s": 4.440309538000292,
"t_B_s": 4.539836316998844,
"t_total_s": 4.540553365997766,
"cached": 32752,
"mode": "layerwise",
"size": 32768,
"rep": 2,
"t_prefill_only_s": 4.443476915999781,
"kv_gib": 3.0,
"correct": true
}
],
"summary": [
{
"size": 8192,
"n": 3,
"pf_only_ms": 589.0174580017629,
"total_ms": 682.8304180016858,
"overhead_ms": 93.8129599999229,
"all_correct": true
},
{
"size": 16384,
"n": 3,
"pf_only_ms": 1513.0829510017065,
"total_ms": 1595.71184500237,
"overhead_ms": 82.62889400066342,
"all_correct": true
},
{
"size": 32768,
"n": 3,
"pf_only_ms": 4445.38704000297,
"total_ms": 4540.553365997766,
"overhead_ms": 95.16632599479635,
"all_correct": true
}
]
}