Gahow Wang
837df6bc9e
Measures TTFT to serve a reused prefix of length L from each KV tier on a single H20 (Qwen3-Coder-30B-A3B, vLLM 0.18.1): miss (recompute), CPU-tier hit (native DRAM offload), GPU-tier hit (HBM prefix cache). Each measured request is bracketed by /metrics scrapes so the tier is verified (vllm:prefix_cache_hits vs external_prefix_cache_hits), not assumed. Result: GPU hit is ~flat (42->111 ms over 1k->64k tokens); CPU hit is transfer-bound (PCIe H2D ~54 GB/s, 57->272 ms); miss grows superlinearly (78 ms -> 15.2 s). GPU beats CPU 1.4-2.5x (gap grows with context); miss/CPU up to 56x, miss/GPU up to 137x. pcie_transfer.py is the independent CPU-hit floor backstop. Evidence for the GPU-hit-first principle (paper section 2.2). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
309 lines
7.6 KiB
JSON
309 lines
7.6 KiB
JSON
{
|
|
"mode": "cpu",
|
|
"reps": 4,
|
|
"by_length": {
|
|
"1024": {
|
|
"n": 4,
|
|
"ttft_p50": 0.057389369496377185,
|
|
"ttft_mean": 0.08805505199416075,
|
|
"ttft_min": 0.055113587994128466,
|
|
"ttft_max": 0.18232788098976016,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"2048": {
|
|
"n": 4,
|
|
"ttft_p50": 0.05680296401260421,
|
|
"ttft_mean": 0.05688378225750057,
|
|
"ttft_min": 0.05582832600339316,
|
|
"ttft_max": 0.05810087500140071,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"4096": {
|
|
"n": 4,
|
|
"ttft_p50": 0.06417885900009423,
|
|
"ttft_mean": 0.06439992749801604,
|
|
"ttft_min": 0.063741421996383,
|
|
"ttft_max": 0.0655005699954927,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"8192": {
|
|
"n": 4,
|
|
"ttft_p50": 0.07612077200610656,
|
|
"ttft_mean": 0.07600563450250775,
|
|
"ttft_min": 0.07499952000216581,
|
|
"ttft_max": 0.07678147399565205,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"16384": {
|
|
"n": 4,
|
|
"ttft_p50": 0.10476256850233767,
|
|
"ttft_mean": 0.10426848525821697,
|
|
"ttft_min": 0.10178845902555622,
|
|
"ttft_max": 0.10576034500263631,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"32768": {
|
|
"n": 4,
|
|
"ttft_p50": 0.15755228500347584,
|
|
"ttft_mean": 0.15822456649766536,
|
|
"ttft_min": 0.15535229499801062,
|
|
"ttft_max": 0.16244140098569915,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
},
|
|
"65536": {
|
|
"n": 4,
|
|
"ttft_p50": 0.27175235451431945,
|
|
"ttft_mean": 0.2716009145078715,
|
|
"ttft_min": 0.26732781299506314,
|
|
"ttft_max": 0.27557113600778393,
|
|
"tier_observed": "cpu",
|
|
"verified_frac": 1.0
|
|
}
|
|
},
|
|
"raw": {
|
|
"1024": [
|
|
{
|
|
"ttft_s": 0.18232788098976016,
|
|
"e2e_s": 0.182449893996818,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 1024.0
|
|
},
|
|
{
|
|
"ttft_s": 0.05692533400724642,
|
|
"e2e_s": 0.057079910999163985,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 1024.0
|
|
},
|
|
{
|
|
"ttft_s": 0.05785340498550795,
|
|
"e2e_s": 0.05796545499470085,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 1024.0
|
|
},
|
|
{
|
|
"ttft_s": 0.055113587994128466,
|
|
"e2e_s": 0.055113587994128466,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 1024.0
|
|
}
|
|
],
|
|
"2048": [
|
|
{
|
|
"ttft_s": 0.05582832600339316,
|
|
"e2e_s": 0.055943820014363155,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 2048.0
|
|
},
|
|
{
|
|
"ttft_s": 0.057600113010266796,
|
|
"e2e_s": 0.05772249499568716,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 2048.0
|
|
},
|
|
{
|
|
"ttft_s": 0.05600581501494162,
|
|
"e2e_s": 0.05611848901025951,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 2048.0
|
|
},
|
|
{
|
|
"ttft_s": 0.05810087500140071,
|
|
"e2e_s": 0.0582130889815744,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 2048.0
|
|
}
|
|
],
|
|
"4096": [
|
|
{
|
|
"ttft_s": 0.0655005699954927,
|
|
"e2e_s": 0.06560429997625761,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 4096.0
|
|
},
|
|
{
|
|
"ttft_s": 0.063741421996383,
|
|
"e2e_s": 0.06384500698186457,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 4096.0
|
|
},
|
|
{
|
|
"ttft_s": 0.06445824800175615,
|
|
"e2e_s": 0.06458494698745199,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 4096.0
|
|
},
|
|
{
|
|
"ttft_s": 0.06389946999843232,
|
|
"e2e_s": 0.06403137900633737,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 4096.0
|
|
}
|
|
],
|
|
"8192": [
|
|
{
|
|
"ttft_s": 0.0759067680046428,
|
|
"e2e_s": 0.0759067680046428,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 8192.0
|
|
},
|
|
{
|
|
"ttft_s": 0.07678147399565205,
|
|
"e2e_s": 0.07678147399565205,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 8192.0
|
|
},
|
|
{
|
|
"ttft_s": 0.07633477600757033,
|
|
"e2e_s": 0.0764636590029113,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 8192.0
|
|
},
|
|
{
|
|
"ttft_s": 0.07499952000216581,
|
|
"e2e_s": 0.07499952000216581,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 8192.0
|
|
}
|
|
],
|
|
"16384": [
|
|
{
|
|
"ttft_s": 0.10444335200008936,
|
|
"e2e_s": 0.104556644015247,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 16384.0
|
|
},
|
|
{
|
|
"ttft_s": 0.10178845902555622,
|
|
"e2e_s": 0.10188649001065642,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 16384.0
|
|
},
|
|
{
|
|
"ttft_s": 0.10508178500458598,
|
|
"e2e_s": 0.10518974298611283,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 16384.0
|
|
},
|
|
{
|
|
"ttft_s": 0.10576034500263631,
|
|
"e2e_s": 0.10587632199167274,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 16384.0
|
|
}
|
|
],
|
|
"32768": [
|
|
{
|
|
"ttft_s": 0.15535229499801062,
|
|
"e2e_s": 0.155460513982689,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 32768.0
|
|
},
|
|
{
|
|
"ttft_s": 0.15777600501314737,
|
|
"e2e_s": 0.15777600501314737,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 32768.0
|
|
},
|
|
{
|
|
"ttft_s": 0.16244140098569915,
|
|
"e2e_s": 0.16244140098569915,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 32768.0
|
|
},
|
|
{
|
|
"ttft_s": 0.15732856499380432,
|
|
"e2e_s": 0.1574467390018981,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 32768.0
|
|
}
|
|
],
|
|
"65536": [
|
|
{
|
|
"ttft_s": 0.27309533301740885,
|
|
"e2e_s": 0.273235183005454,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 65536.0
|
|
},
|
|
{
|
|
"ttft_s": 0.27040937601123005,
|
|
"e2e_s": 0.27040937601123005,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 65536.0
|
|
},
|
|
{
|
|
"ttft_s": 0.27557113600778393,
|
|
"e2e_s": 0.27557113600778393,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 65536.0
|
|
},
|
|
{
|
|
"ttft_s": 0.26732781299506314,
|
|
"e2e_s": 0.2674778919899836,
|
|
"tier_observed": "cpu",
|
|
"expect": "cpu",
|
|
"d_gpu_hits": 0.0,
|
|
"d_ext_hits": 65536.0
|
|
}
|
|
]
|
|
}
|
|
} |