gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v2 exp(a): three-tier KV-hit latency microbench (GPU >> CPU >> miss)

Browse Source 
Measures TTFT to serve a reused prefix of length L from each KV tier on a
single H20 (Qwen3-Coder-30B-A3B, vLLM 0.18.1): miss (recompute), CPU-tier
hit (native DRAM offload), GPU-tier hit (HBM prefix cache). Each measured
request is bracketed by /metrics scrapes so the tier is verified
(vllm:prefix_cache_hits vs external_prefix_cache_hits), not assumed.

Result: GPU hit is ~flat (42->111 ms over 1k->64k tokens); CPU hit is
transfer-bound (PCIe H2D ~54 GB/s, 57->272 ms); miss grows superlinearly
(78 ms -> 15.2 s). GPU beats CPU 1.4-2.5x (gap grows with context);
miss/CPU up to 56x, miss/GPU up to 137x. pcie_transfer.py is the
independent CPU-hit floor backstop. Evidence for the GPU-hit-first
principle (paper section 2.2).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-30 11:23:04 +08:00
parent cf812b6264
commit 837df6bc9e
12 changed files with 1880 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
v2/exp_a_tier_latency/results/cpu.json Normal file
View File

@@ -0,0 +1,309 @@
{
"mode": "cpu",
"reps": 4,
"by_length": {
"1024": {
"n": 4,
"ttft_p50": 0.057389369496377185,
"ttft_mean": 0.08805505199416075,
"ttft_min": 0.055113587994128466,
"ttft_max": 0.18232788098976016,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"2048": {
"n": 4,
"ttft_p50": 0.05680296401260421,
"ttft_mean": 0.05688378225750057,
"ttft_min": 0.05582832600339316,
"ttft_max": 0.05810087500140071,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"4096": {
"n": 4,
"ttft_p50": 0.06417885900009423,
"ttft_mean": 0.06439992749801604,
"ttft_min": 0.063741421996383,
"ttft_max": 0.0655005699954927,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"8192": {
"n": 4,
"ttft_p50": 0.07612077200610656,
"ttft_mean": 0.07600563450250775,
"ttft_min": 0.07499952000216581,
"ttft_max": 0.07678147399565205,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"16384": {
"n": 4,
"ttft_p50": 0.10476256850233767,
"ttft_mean": 0.10426848525821697,
"ttft_min": 0.10178845902555622,
"ttft_max": 0.10576034500263631,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"32768": {
"n": 4,
"ttft_p50": 0.15755228500347584,
"ttft_mean": 0.15822456649766536,
"ttft_min": 0.15535229499801062,
"ttft_max": 0.16244140098569915,
"tier_observed": "cpu",
"verified_frac": 1.0
},
"65536": {
"n": 4,
"ttft_p50": 0.27175235451431945,
"ttft_mean": 0.2716009145078715,
"ttft_min": 0.26732781299506314,
"ttft_max": 0.27557113600778393,
"tier_observed": "cpu",
"verified_frac": 1.0
}
},
"raw": {
"1024": [
{
"ttft_s": 0.18232788098976016,
"e2e_s": 0.182449893996818,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 1024.0
},
{
"ttft_s": 0.05692533400724642,
"e2e_s": 0.057079910999163985,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 1024.0
},
{
"ttft_s": 0.05785340498550795,
"e2e_s": 0.05796545499470085,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 1024.0
},
{
"ttft_s": 0.055113587994128466,
"e2e_s": 0.055113587994128466,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 1024.0
}
],
"2048": [
{
"ttft_s": 0.05582832600339316,
"e2e_s": 0.055943820014363155,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 2048.0
},
{
"ttft_s": 0.057600113010266796,
"e2e_s": 0.05772249499568716,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 2048.0
},
{
"ttft_s": 0.05600581501494162,
"e2e_s": 0.05611848901025951,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 2048.0
},
{
"ttft_s": 0.05810087500140071,
"e2e_s": 0.0582130889815744,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 2048.0
}
],
"4096": [
{
"ttft_s": 0.0655005699954927,
"e2e_s": 0.06560429997625761,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 4096.0
},
{
"ttft_s": 0.063741421996383,
"e2e_s": 0.06384500698186457,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 4096.0
},
{
"ttft_s": 0.06445824800175615,
"e2e_s": 0.06458494698745199,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 4096.0
},
{
"ttft_s": 0.06389946999843232,
"e2e_s": 0.06403137900633737,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 4096.0
}
],
"8192": [
{
"ttft_s": 0.0759067680046428,
"e2e_s": 0.0759067680046428,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 8192.0
},
{
"ttft_s": 0.07678147399565205,
"e2e_s": 0.07678147399565205,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 8192.0
},
{
"ttft_s": 0.07633477600757033,
"e2e_s": 0.0764636590029113,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 8192.0
},
{
"ttft_s": 0.07499952000216581,
"e2e_s": 0.07499952000216581,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 8192.0
}
],
"16384": [
{
"ttft_s": 0.10444335200008936,
"e2e_s": 0.104556644015247,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 16384.0
},
{
"ttft_s": 0.10178845902555622,
"e2e_s": 0.10188649001065642,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 16384.0
},
{
"ttft_s": 0.10508178500458598,
"e2e_s": 0.10518974298611283,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 16384.0
},
{
"ttft_s": 0.10576034500263631,
"e2e_s": 0.10587632199167274,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 16384.0
}
],
"32768": [
{
"ttft_s": 0.15535229499801062,
"e2e_s": 0.155460513982689,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 32768.0
},
{
"ttft_s": 0.15777600501314737,
"e2e_s": 0.15777600501314737,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 32768.0
},
{
"ttft_s": 0.16244140098569915,
"e2e_s": 0.16244140098569915,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 32768.0
},
{
"ttft_s": 0.15732856499380432,
"e2e_s": 0.1574467390018981,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 32768.0
}
],
"65536": [
{
"ttft_s": 0.27309533301740885,
"e2e_s": 0.273235183005454,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 65536.0
},
{
"ttft_s": 0.27040937601123005,
"e2e_s": 0.27040937601123005,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 65536.0
},
{
"ttft_s": 0.27557113600778393,
"e2e_s": 0.27557113600778393,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 65536.0
},
{
"ttft_s": 0.26732781299506314,
"e2e_s": 0.2674778919899836,
"tier_observed": "cpu",
"expect": "cpu",
"d_gpu_hits": 0.0,
"d_ext_hits": 65536.0
}
]
}
}

533
v2/exp_a_tier_latency/results/gpu.json Normal file
View File

@@ -0,0 +1,533 @@
{
"mode": "gpu",
"reps": 8,
"by_length": {
"1024": {
"n": 8,
"ttft_p50": 0.04180275100225117,
"ttft_mean": 0.05689269150025211,
"ttft_min": 0.041313502995762974,
"ttft_max": 0.1606091230059974,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"2048": {
"n": 8,
"ttft_p50": 0.044922845510882325,
"ttft_mean": 0.04646045462868642,
"ttft_min": 0.04261300901998766,
"ttft_max": 0.06082483098725788,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"4096": {
"n": 8,
"ttft_p50": 0.0462174300046172,
"ttft_mean": 0.04691218675361597,
"ttft_min": 0.044408742018276826,
"ttft_max": 0.05101387499598786,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"8192": {
"n": 8,
"ttft_p50": 0.052487702007056214,
"ttft_mean": 0.05252782000388834,
"ttft_min": 0.050384567002765834,
"ttft_max": 0.055209266021847725,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"16384": {
"n": 8,
"ttft_p50": 0.06340778700541705,
"ttft_mean": 0.06307360512437299,
"ttft_min": 0.059953891002805904,
"ttft_max": 0.06587072199909016,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"32768": {
"n": 8,
"ttft_p50": 0.07986902150150854,
"ttft_mean": 0.08412684850554797,
"ttft_min": 0.07615292401169427,
"ttft_max": 0.11761908099288121,
"tier_observed": "gpu",
"verified_frac": 1.0
},
"65536": {
"n": 8,
"ttft_p50": 0.11140661900572013,
"ttft_mean": 0.10751268000240088,
"ttft_min": 0.07390080401091836,
"ttft_max": 0.1206158839922864,
"tier_observed": "gpu",
"verified_frac": 1.0
}
},
"raw": {
"1024": [
{
"ttft_s": 0.1606091230059974,
"e2e_s": 0.16078226300305687,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.043477901024743915,
"e2e_s": 0.0436010490229819,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04137404798530042,
"e2e_s": 0.04146770399529487,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04179324599681422,
"e2e_s": 0.041887808009050786,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04326947100344114,
"e2e_s": 0.04335355598595925,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04149198398226872,
"e2e_s": 0.04157822398701683,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04181225600768812,
"e2e_s": 0.04190706100780517,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.041313502995762974,
"e2e_s": 0.041313502995762974,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 1008.0,
"d_ext_hits": 0.0
}
],
"2048": [
{
"ttft_s": 0.04491939002764411,
"e2e_s": 0.045019031007541344,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.045014784001978114,
"e2e_s": 0.04511277299025096,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04492630099412054,
"e2e_s": 0.04502850098651834,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04566141500254162,
"e2e_s": 0.04576313399593346,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04261300901998766,
"e2e_s": 0.04271370900096372,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06082483098725788,
"e2e_s": 0.06096197199076414,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04355804901570082,
"e2e_s": 0.04355804901570082,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.044165857980260625,
"e2e_s": 0.044268568977713585,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 2032.0,
"d_ext_hits": 0.0
}
],
"4096": [
{
"ttft_s": 0.05101387499598786,
"e2e_s": 0.051123478973750025,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.044408742018276826,
"e2e_s": 0.044408742018276826,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04538871700060554,
"e2e_s": 0.045498208986828104,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04754545699688606,
"e2e_s": 0.047664124984294176,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04840670898556709,
"e2e_s": 0.04840670898556709,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0462190090038348,
"e2e_s": 0.04632823000429198,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.04609913402236998,
"e2e_s": 0.046204126003431156,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0462158510053996,
"e2e_s": 0.0462158510053996,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 4080.0,
"d_ext_hits": 0.0
}
],
"8192": [
{
"ttft_s": 0.05042222701013088,
"e2e_s": 0.05053543800022453,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.05319672200130299,
"e2e_s": 0.053308423986891285,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.05063424099353142,
"e2e_s": 0.05073276098119095,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.050384567002765834,
"e2e_s": 0.05048462699051015,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.055105848994571716,
"e2e_s": 0.055215683998540044,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.05349100599414669,
"e2e_s": 0.053595816978486255,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.05177868201280944,
"e2e_s": 0.05188246400211938,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.055209266021847725,
"e2e_s": 0.05531894601881504,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 8176.0,
"d_ext_hits": 0.0
}
],
"16384": [
{
"ttft_s": 0.0633803239907138,
"e2e_s": 0.06349112599855289,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06337857199832797,
"e2e_s": 0.06350608498905785,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06098292299429886,
"e2e_s": 0.061115075019188225,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06343525002012029,
"e2e_s": 0.06355450401315466,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0636955969966948,
"e2e_s": 0.0636955969966948,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06389156199293211,
"e2e_s": 0.06389156199293211,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.06587072199909016,
"e2e_s": 0.06587072199909016,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.059953891002805904,
"e2e_s": 0.060058912000386044,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 16368.0,
"d_ext_hits": 0.0
}
],
"32768": [
{
"ttft_s": 0.07615292401169427,
"e2e_s": 0.07625289200223051,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07992992899380624,
"e2e_s": 0.0800386439950671,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07980811400921084,
"e2e_s": 0.07995001602103002,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.11761908099288121,
"e2e_s": 0.11776423300034367,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07834753501811065,
"e2e_s": 0.07834753501811065,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0814115820103325,
"e2e_s": 0.0814115820103325,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.08212830501724966,
"e2e_s": 0.08224253499065526,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07761731799109839,
"e2e_s": 0.07772363899857737,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 32752.0,
"d_ext_hits": 0.0
}
],
"65536": [
{
"ttft_s": 0.1206158839922864,
"e2e_s": 0.1206158839922864,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.10727833199780434,
"e2e_s": 0.10727833199780434,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.11010084400186315,
"e2e_s": 0.11023741000099108,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.11566799599677324,
"e2e_s": 0.1157765949901659,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.11422122400836088,
"e2e_s": 0.11422122400836088,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07390080401091836,
"e2e_s": 0.07390080401091836,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.11271239400957711,
"e2e_s": 0.11271239400957711,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.10560396200162359,
"e2e_s": 0.10572021701955236,
"tier_observed": "gpu",
"expect": "gpu",
"d_gpu_hits": 65520.0,
"d_ext_hits": 0.0
}
]
}
}

533
v2/exp_a_tier_latency/results/miss.json Normal file
View File

@@ -0,0 +1,533 @@
{
"mode": "miss",
"reps": 8,
"by_length": {
"1024": {
"n": 8,
"ttft_p50": 0.07843716802017298,
"ttft_mean": 0.1441972145003092,
"ttft_min": 0.0768489159818273,
"ttft_max": 0.6051040079910308,
"tier_observed": "miss",
"verified_frac": 1.0
},
"2048": {
"n": 8,
"ttft_p50": 0.13254689799214248,
"ttft_mean": 0.14012471562818973,
"ttft_min": 0.13148935200297274,
"ttft_max": 0.19460841300315224,
"tier_observed": "miss",
"verified_frac": 1.0
},
"4096": {
"n": 8,
"ttft_p50": 0.2613626064994605,
"ttft_mean": 0.2620435417484259,
"ttft_min": 0.26085699602845125,
"ttft_max": 0.2652779980271589,
"tier_observed": "miss",
"verified_frac": 1.0
},
"8192": {
"n": 8,
"ttft_p50": 0.588181210012408,
"ttft_mean": 0.5881437246280257,
"ttft_min": 0.5867919930024073,
"ttft_max": 0.5897468629991636,
"tier_observed": "miss",
"verified_frac": 1.0
},
"16384": {
"n": 8,
"ttft_p50": 1.5470821364870062,
"ttft_mean": 1.5479571051182575,
"ttft_min": 1.5448924789961893,
"ttft_max": 1.5552692519850098,
"tier_observed": "miss",
"verified_frac": 1.0
},
"32768": {
"n": 8,
"ttft_p50": 4.603862981006387,
"ttft_mean": 4.601982127005613,
"ttft_min": 4.594119774992578,
"ttft_max": 4.608500114001799,
"tier_observed": "miss",
"verified_frac": 1.0
},
"65536": {
"n": 8,
"ttft_p50": 15.230140178493457,
"ttft_mean": 15.229316346121777,
"ttft_min": 15.177009812992765,
"ttft_max": 15.28301460199873,
"tier_observed": "miss",
"verified_frac": 1.0
}
},
"raw": {
"1024": [
{
"ttft_s": 0.6051040079910308,
"e2e_s": 0.6052766389911994,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.08014080999419093,
"e2e_s": 0.08025239198468626,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07735737299663015,
"e2e_s": 0.07746260898420587,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.0768489159818273,
"e2e_s": 0.0768489159818273,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07845149302738719,
"e2e_s": 0.07860825702664442,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07842284301295877,
"e2e_s": 0.07842284301295877,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.07716230300138704,
"e2e_s": 0.07729722000658512,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.08008996999706142,
"e2e_s": 0.08020236299489625,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"2048": [
{
"ttft_s": 0.19460841300315224,
"e2e_s": 0.19472959099221043,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13326648401562124,
"e2e_s": 0.13339112501125783,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.1324451889959164,
"e2e_s": 0.13255105601274408,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13292257199645974,
"e2e_s": 0.1330343289882876,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13170140300644562,
"e2e_s": 0.13180866098264232,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13264860698836856,
"e2e_s": 0.13274579899734817,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13148935200297274,
"e2e_s": 0.13160509002045728,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.13191570501658134,
"e2e_s": 0.1320405500009656,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"4096": [
{
"ttft_s": 0.2652779980271589,
"e2e_s": 0.2653735490166582,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26092263497412205,
"e2e_s": 0.2610465929901693,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26085699602845125,
"e2e_s": 0.2609657910070382,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.2613706949923653,
"e2e_s": 0.2613706949923653,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26135451800655574,
"e2e_s": 0.2614478030009195,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.2610389889741782,
"e2e_s": 0.2611535779724363,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26337735800188966,
"e2e_s": 0.2634820609819144,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.26214914498268627,
"e2e_s": 0.26226521999342367,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"8192": [
{
"ttft_s": 0.5897468629991636,
"e2e_s": 0.5898835949774366,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5884731799887959,
"e2e_s": 0.5884731799887959,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.588430589006748,
"e2e_s": 0.588430589006748,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5895502530038357,
"e2e_s": 0.5896741840115283,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5879318310180679,
"e2e_s": 0.5879318310180679,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5870601140195504,
"e2e_s": 0.5871822330227587,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5871649739856366,
"e2e_s": 0.5871649739856366,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 0.5867919930024073,
"e2e_s": 0.5869127069890965,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"16384": [
{
"ttft_s": 1.5552692519850098,
"e2e_s": 1.5552692519850098,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5471642419870477,
"e2e_s": 1.547288056986872,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5476988689915743,
"e2e_s": 1.5476988689915743,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5459686409740243,
"e2e_s": 1.5460858139849734,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.548691272008,
"e2e_s": 1.548691272008,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5470000309869647,
"e2e_s": 1.5471212370030116,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5448924789961893,
"e2e_s": 1.5448924789961893,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 1.5469720550172497,
"e2e_s": 1.5470902820234187,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"32768": [
{
"ttft_s": 4.603710585011868,
"e2e_s": 4.603829422005219,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.608500114001799,
"e2e_s": 4.608500114001799,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.604015377000906,
"e2e_s": 4.604134508001152,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.606255694001447,
"e2e_s": 4.606392626999877,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.598494538018713,
"e2e_s": 4.598494538018713,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.594119774992578,
"e2e_s": 4.59425267498591,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.594318103016121,
"e2e_s": 4.594462227018084,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 4.60644283000147,
"e2e_s": 4.60644283000147,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
],
"65536": [
{
"ttft_s": 15.276076515991008,
"e2e_s": 15.276189939002506,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.275313204998383,
"e2e_s": 15.275428983004531,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.274311708984897,
"e2e_s": 15.274442903988529,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.28301460199873,
"e2e_s": 15.283127713017166,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.185968648002017,
"e2e_s": 15.185968648002017,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.177009812992765,
"e2e_s": 15.177117884973995,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.181360610004049,
"e2e_s": 15.181482268002583,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
},
{
"ttft_s": 15.18147566600237,
"e2e_s": 15.18147566600237,
"tier_observed": "miss",
"expect": "miss",
"d_gpu_hits": 0.0,
"d_ext_hits": 0.0
}
]
}
}

40
v2/exp_a_tier_latency/results/pcie.json Normal file
View File

@@ -0,0 +1,40 @@
{
"device": "NVIDIA H20",
"by_length": {
"1024": {
"kv_bytes": 100663296,
"transfer_s": 0.001876260997960344,
"bw_GBps": 53.65100916633112
},
"2048": {
"kv_bytes": 201326592,
"transfer_s": 0.003709116979734972,
"bw_GBps": 54.27884671741612
},
"4096": {
"kv_bytes": 402653184,
"transfer_s": 0.007338636991335079,
"bw_GBps": 54.86757070494469
},
"8192": {
"kv_bytes": 805306368,
"transfer_s": 0.01476299500791356,
"bw_GBps": 54.548983290201164
},
"16384": {
"kv_bytes": 1610612736,
"transfer_s": 0.02972855800180696,
"bw_GBps": 54.17729093695375
},
"32768": {
"kv_bytes": 3221225472,
"transfer_s": 0.059267577016726136,
"bw_GBps": 54.35055107940257
},
"65536": {
"kv_bytes": 6442450944,
"transfer_s": 0.11847134301206097,
"bw_GBps": 54.37982536708583
}
}
}