gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
837df6bc9e05f5a5b4af27a36356efdd4cff2ecd
agentic-kvc/v2/exp_a_tier_latency/results/miss.json
Gahow Wang 837df6bc9e v2 exp(a): three-tier KV-hit latency microbench (GPU >> CPU >> miss) 
Measures TTFT to serve a reused prefix of length L from each KV tier on a
single H20 (Qwen3-Coder-30B-A3B, vLLM 0.18.1): miss (recompute), CPU-tier
hit (native DRAM offload), GPU-tier hit (HBM prefix cache). Each measured
request is bracketed by /metrics scrapes so the tier is verified
(vllm:prefix_cache_hits vs external_prefix_cache_hits), not assumed.

Result: GPU hit is ~flat (42->111 ms over 1k->64k tokens); CPU hit is
transfer-bound (PCIe H2D ~54 GB/s, 57->272 ms); miss grows superlinearly
(78 ms -> 15.2 s). GPU beats CPU 1.4-2.5x (gap grows with context);
miss/CPU up to 56x, miss/GPU up to 137x. pcie_transfer.py is the
independent CPU-hit floor backstop. Evidence for the GPU-hit-first
principle (paper section 2.2).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-05-30 11:23:04 +08:00

533 lines
13 KiB
JSON
Raw Blame History

{
"mode": "miss",
"reps": 8,
"by_length": {
"1024": {
"n": 8,
"ttft_p50": 0.07843716802017298,
"ttft_mean": 0.1441972145003092,
"ttft_min": 0.0768489159818273,
"ttft_max": 0.6051040079910308,
"tier_observed": "miss",
"verified_frac": 1.0
},
"2048": {
"n": 8,
"ttft_p50": 0.13254689799214248,
"ttft_mean": 0.14012471562818973,
"ttft_min": 0.13148935200297274,
"ttft_max": 0.19460841300315224,
"tier_observed": "miss",
"verified_frac": 1.0
},
"4096": {
"n": 8,
"ttft_p50": 0.2613626064994605,
"ttft_mean": 0.2620435417484259,
"ttft_min": 0.26085699602845125,
"ttft_max": 0.2652779980271589,
"tier_observed": "miss",
"verified_frac": 1.0
},
"8192": {
"n": 8,
"ttft_p50": 0.588181210012408,
"ttft_mean": 0.5881437246280257,
"ttft_min": 0.5867919930024073,
"ttft_max": 0.5897468629991636,
"tier_observed": "miss",
"verified_frac": 1.0
},
"16384": {
"n": 8,
"ttft_p50": 1.5470821364870062,
"ttft_mean": 1.5479571051182575,
"ttft_min": 1.5448924789961893,
"ttft_max": 1.5552692519850098,
"tier_observed": "miss",
"verified_frac": 1.0
},
"32768": {
"n": 8,
"ttft_p50": 4.603862981006387,
"ttft_mean": 4.601982127005613,
"ttft_min": 4.594119774992578,
"ttft_max": 4.608500114001799,
"tier_observed": "miss",
"verified_frac": 1.0
},
"65536": {
"n": 8,
"ttft_p50": 15.230140178493457,
"ttft_mean": 15.229316346121777,
"ttft_min": 15.177009812992765,
"ttft_max": 15.28301460199873,
"tier_observed": "miss",
"verified_frac": 1.0
}
},
"raw": {
"1024": [
{
"ttft_s": 0.6051040079910308,
"e2e_s": 0.6052766389911994,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.08014080999419093,
"e2e_s": 0.08025239198468626,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07735737299663015,
"e2e_s": 0.07746260898420587,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0768489159818273,
"e2e_s": 0.0768489159818273,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07845149302738719,
"e2e_s": 0.07860825702664442,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07842284301295877,
"e2e_s": 0.07842284301295877,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07716230300138704,
"e2e_s": 0.07729722000658512,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.08008996999706142,
"e2e_s": 0.08020236299489625,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"2048": [
{
"ttft_s": 0.19460841300315224,
"e2e_s": 0.19472959099221043,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13326648401562124,
"e2e_s": 0.13339112501125783,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.1324451889959164,
"e2e_s": 0.13255105601274408,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13292257199645974,
"e2e_s": 0.1330343289882876,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13170140300644562,
"e2e_s": 0.13180866098264232,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13264860698836856,
"e2e_s": 0.13274579899734817,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13148935200297274,
"e2e_s": 0.13160509002045728,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13191570501658134,
"e2e_s": 0.1320405500009656,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"4096": [
{
"ttft_s": 0.2652779980271589,
"e2e_s": 0.2653735490166582,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26092263497412205,
"e2e_s": 0.2610465929901693,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26085699602845125,
"e2e_s": 0.2609657910070382,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.2613706949923653,
"e2e_s": 0.2613706949923653,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26135451800655574,
"e2e_s": 0.2614478030009195,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.2610389889741782,
"e2e_s": 0.2611535779724363,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26337735800188966,
"e2e_s": 0.2634820609819144,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26214914498268627,
"e2e_s": 0.26226521999342367,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"8192": [
{
"ttft_s": 0.5897468629991636,
"e2e_s": 0.5898835949774366,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5884731799887959,
"e2e_s": 0.5884731799887959,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.588430589006748,
"e2e_s": 0.588430589006748,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5895502530038357,
"e2e_s": 0.5896741840115283,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5879318310180679,
"e2e_s": 0.5879318310180679,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5870601140195504,
"e2e_s": 0.5871822330227587,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5871649739856366,
"e2e_s": 0.5871649739856366,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5867919930024073,
"e2e_s": 0.5869127069890965,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"16384": [
{
"ttft_s": 1.5552692519850098,
"e2e_s": 1.5552692519850098,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5471642419870477,
"e2e_s": 1.547288056986872,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5476988689915743,
"e2e_s": 1.5476988689915743,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5459686409740243,
"e2e_s": 1.5460858139849734,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.548691272008,
"e2e_s": 1.548691272008,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5470000309869647,
"e2e_s": 1.5471212370030116,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5448924789961893,
"e2e_s": 1.5448924789961893,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5469720550172497,
"e2e_s": 1.5470902820234187,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"32768": [
{
"ttft_s": 4.603710585011868,
"e2e_s": 4.603829422005219,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.608500114001799,
"e2e_s": 4.608500114001799,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.604015377000906,
"e2e_s": 4.604134508001152,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.606255694001447,
"e2e_s": 4.606392626999877,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.598494538018713,
"e2e_s": 4.598494538018713,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.594119774992578,
"e2e_s": 4.59425267498591,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.594318103016121,
"e2e_s": 4.594462227018084,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.60644283000147,
"e2e_s": 4.60644283000147,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"65536": [
{
"ttft_s": 15.276076515991008,
"e2e_s": 15.276189939002506,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.275313204998383,
"e2e_s": 15.275428983004531,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.274311708984897,
"e2e_s": 15.274442903988529,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.28301460199873,
"e2e_s": 15.283127713017166,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.185968648002017,
"e2e_s": 15.185968648002017,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.177009812992765,
"e2e_s": 15.177117884973995,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.181360610004049,
"e2e_s": 15.181482268002583,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.18147566600237,
"e2e_s": 15.18147566600237,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
]
}
}
Reference in New Issue View Git Blame Copy Permalink