{ "capacity": [ { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 8.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 8.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 8.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 8.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 16.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 16.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 16.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 16.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 32.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 32.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 32.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "lower_censored": false, "maximum_tested_feasible_request_rate": 32.0, "maximum_tested_feasible_request_rate_per_gpu": 8.0, "upper_censored": false } ], "collective": { "allreduce_csv": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", "allreduce_csv_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", "backend": "vidur" }, "config_results": [ { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "loads": [ { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "elapsed_seconds": 157.35246539115906, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.020947591286289, "ttft_max_ms": 166.6461504479848, "ttft_p50_ms": 166.64615044798398, "ttft_p95_ms": 166.6461504479848 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "elapsed_seconds": 9.141019344329834, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 7.75274759930899, "ttft_max_ms": 586.937565208359, "ttft_p50_ms": 436.98066306434936, "ttft_p95_ms": 575.6631856791392 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "elapsed_seconds": 9.098863363265991, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", "score": { "feasible": false, "pass_rate": 0.265625, "passed": 17, "throughput_requests_per_second": 8.523874834682788, "ttft_max_ms": 3654.1762077816857, "ttft_p50_ms": 1994.5136762864174, "ttft_p95_ms": 3529.1762077816857 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "elapsed_seconds": 9.089977025985718, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", "score": { "feasible": false, "pass_rate": 0.1875, "passed": 12, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 5483.402803563629, "ttft_p50_ms": 2911.6176150140045, "ttft_p95_ms": 5290.0301465092725 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 8, "name": "tp1_mns8", "tp": 1 }, "elapsed_seconds": 9.088061094284058, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 64.0, "request_count": 64, "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", "score": { "feasible": false, "pass_rate": 0.140625, "passed": 9, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 6436.527803563629, "ttft_p50_ms": 3380.3676150140045, "ttft_p95_ms": 6180.6551465092725 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "loads": [ { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "elapsed_seconds": 8.947673320770264, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.020947591286289, "ttft_max_ms": 166.6461504479848, "ttft_p50_ms": 166.64615044798398, "ttft_p95_ms": 166.6461504479848 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "elapsed_seconds": 9.042973518371582, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 7.75274759930899, "ttft_max_ms": 586.937565208359, "ttft_p50_ms": 436.98066306434936, "ttft_p95_ms": 575.6631856791392 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "elapsed_seconds": 9.092506170272827, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", "score": { "feasible": false, "pass_rate": 0.265625, "passed": 17, "throughput_requests_per_second": 8.523874834682788, "ttft_max_ms": 3654.1762077816857, "ttft_p50_ms": 1994.5136762864174, "ttft_p95_ms": 3529.1762077816857 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "elapsed_seconds": 9.044708013534546, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", "score": { "feasible": false, "pass_rate": 0.1875, "passed": 12, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 5483.402803563629, "ttft_p50_ms": 2911.6176150140045, "ttft_p95_ms": 5290.0301465092725 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 16, "name": "tp1_mns16", "tp": 1 }, "elapsed_seconds": 9.29175853729248, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 64.0, "request_count": 64, "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", "score": { "feasible": false, "pass_rate": 0.140625, "passed": 9, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 6436.527803563629, "ttft_p50_ms": 3380.3676150140045, "ttft_p95_ms": 6180.6551465092725 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "loads": [ { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "elapsed_seconds": 8.991463661193848, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.020947591286289, "ttft_max_ms": 166.6461504479848, "ttft_p50_ms": 166.64615044798398, "ttft_p95_ms": 166.6461504479848 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "elapsed_seconds": 9.138724327087402, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 7.75274759930899, "ttft_max_ms": 586.937565208359, "ttft_p50_ms": 436.98066306434936, "ttft_p95_ms": 575.6631856791392 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "elapsed_seconds": 9.188970565795898, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", "score": { "feasible": false, "pass_rate": 0.265625, "passed": 17, "throughput_requests_per_second": 8.523874834682788, "ttft_max_ms": 3654.1762077816857, "ttft_p50_ms": 1994.5136762864174, "ttft_p95_ms": 3529.1762077816857 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "elapsed_seconds": 9.189127683639526, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", "score": { "feasible": false, "pass_rate": 0.1875, "passed": 12, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 5483.402803563629, "ttft_p50_ms": 2911.6176150140045, "ttft_p95_ms": 5290.0301465092725 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 32, "name": "tp1_mns32", "tp": 1 }, "elapsed_seconds": 9.189879894256592, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 64.0, "request_count": 64, "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", "score": { "feasible": false, "pass_rate": 0.140625, "passed": 9, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 6436.527803563629, "ttft_p50_ms": 3380.3676150140045, "ttft_p95_ms": 6180.6551465092725 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "loads": [ { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "elapsed_seconds": 8.999529123306274, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.020947591286289, "ttft_max_ms": 166.6461504479848, "ttft_p50_ms": 166.64615044798398, "ttft_p95_ms": 166.6461504479848 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "elapsed_seconds": 9.192278385162354, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 7.75274759930899, "ttft_max_ms": 586.937565208359, "ttft_p50_ms": 436.98066306434936, "ttft_p95_ms": 575.6631856791392 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "elapsed_seconds": 9.040328979492188, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", "score": { "feasible": false, "pass_rate": 0.265625, "passed": 17, "throughput_requests_per_second": 8.523874834682788, "ttft_max_ms": 3654.1762077816857, "ttft_p50_ms": 1994.5136762864174, "ttft_p95_ms": 3529.1762077816857 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "elapsed_seconds": 8.990798473358154, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", "score": { "feasible": false, "pass_rate": 0.1875, "passed": 12, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 5483.402803563629, "ttft_p50_ms": 2911.6176150140045, "ttft_p95_ms": 5290.0301465092725 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 64, "name": "tp1_mns64", "tp": 1 }, "elapsed_seconds": 9.192291736602783, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 64.0, "request_count": 64, "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", "score": { "feasible": false, "pass_rate": 0.140625, "passed": 9, "throughput_requests_per_second": 8.660758725922316, "ttft_max_ms": 6436.527803563629, "ttft_p50_ms": 3380.3676150140045, "ttft_p95_ms": 6180.6551465092725 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "loads": [ { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "elapsed_seconds": 9.093063116073608, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.033134335322499, "ttft_max_ms": 118.55152318709017, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318709017 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "elapsed_seconds": 160.23531913757324, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.006453678862723, "ttft_max_ms": 118.55152318708929, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318708929 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "elapsed_seconds": 9.289193630218506, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 13.760059242174107, "ttft_max_ms": 849.4875986853465, "ttft_p50_ms": 539.7284419614418, "ttft_p95_ms": 786.9875986853465 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "elapsed_seconds": 9.239543676376343, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", "score": { "feasible": false, "pass_rate": 0.421875, "passed": 27, "throughput_requests_per_second": 14.110410219984, "ttft_max_ms": 2660.658354521784, "ttft_p50_ms": 1414.3128122232463, "ttft_p95_ms": 2566.908354521784 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 8, "name": "tp2_mns8", "tp": 2 }, "elapsed_seconds": 9.141466856002808, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", "score": { "feasible": false, "pass_rate": 0.3125, "passed": 20, "throughput_requests_per_second": 14.110410219983999, "ttft_max_ms": 3582.533354521785, "ttft_p50_ms": 1896.1470654856275, "ttft_p95_ms": 3439.824414996849 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "loads": [ { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "elapsed_seconds": 9.042891025543213, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.033134335322499, "ttft_max_ms": 118.55152318709017, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318709017 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "elapsed_seconds": 9.136998653411865, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.006453678862723, "ttft_max_ms": 118.55152318708929, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318708929 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "elapsed_seconds": 9.13957166671753, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 13.760059242174107, "ttft_max_ms": 849.4875986853465, "ttft_p50_ms": 539.7284419614418, "ttft_p95_ms": 786.9875986853465 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "elapsed_seconds": 9.098120927810669, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", "score": { "feasible": false, "pass_rate": 0.421875, "passed": 27, "throughput_requests_per_second": 14.110410219984, "ttft_max_ms": 2660.658354521784, "ttft_p50_ms": 1414.3128122232463, "ttft_p95_ms": 2566.908354521784 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 16, "name": "tp2_mns16", "tp": 2 }, "elapsed_seconds": 9.140004634857178, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", "score": { "feasible": false, "pass_rate": 0.3125, "passed": 20, "throughput_requests_per_second": 14.110410219983999, "ttft_max_ms": 3582.533354521785, "ttft_p50_ms": 1896.1470654856275, "ttft_p95_ms": 3439.824414996849 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "loads": [ { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "elapsed_seconds": 9.090512752532959, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.033134335322499, "ttft_max_ms": 118.55152318709017, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318709017 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "elapsed_seconds": 9.102084636688232, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.006453678862723, "ttft_max_ms": 118.55152318708929, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318708929 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "elapsed_seconds": 9.09265947341919, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 13.760059242174107, "ttft_max_ms": 849.4875986853465, "ttft_p50_ms": 539.7284419614418, "ttft_p95_ms": 786.9875986853465 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "elapsed_seconds": 9.14569616317749, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", "score": { "feasible": false, "pass_rate": 0.421875, "passed": 27, "throughput_requests_per_second": 14.110410219984, "ttft_max_ms": 2660.658354521784, "ttft_p50_ms": 1414.3128122232463, "ttft_p95_ms": 2566.908354521784 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 32, "name": "tp2_mns32", "tp": 2 }, "elapsed_seconds": 9.191733598709106, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", "score": { "feasible": false, "pass_rate": 0.3125, "passed": 20, "throughput_requests_per_second": 14.110410219983999, "ttft_max_ms": 3582.533354521785, "ttft_p50_ms": 1896.1470654856275, "ttft_p95_ms": 3439.824414996849 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "loads": [ { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "elapsed_seconds": 9.146432876586914, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.033134335322499, "ttft_max_ms": 118.55152318709017, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318709017 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "elapsed_seconds": 9.068153619766235, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.006453678862723, "ttft_max_ms": 118.55152318708929, "ttft_p50_ms": 118.55152318708929, "ttft_p95_ms": 118.55152318708929 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "elapsed_seconds": 9.139062404632568, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 13.760059242174107, "ttft_max_ms": 849.4875986853465, "ttft_p50_ms": 539.7284419614418, "ttft_p95_ms": 786.9875986853465 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "elapsed_seconds": 9.143300771713257, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", "score": { "feasible": false, "pass_rate": 0.421875, "passed": 27, "throughput_requests_per_second": 14.110410219984, "ttft_max_ms": 2660.658354521784, "ttft_p50_ms": 1414.3128122232463, "ttft_p95_ms": 2566.908354521784 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 64, "name": "tp2_mns64", "tp": 2 }, "elapsed_seconds": 9.092010021209717, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 32.0, "request_count": 64, "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", "score": { "feasible": false, "pass_rate": 0.3125, "passed": 20, "throughput_requests_per_second": 14.110410219983999, "ttft_max_ms": 3582.533354521785, "ttft_p50_ms": 1896.1470654856275, "ttft_p95_ms": 3439.824414996849 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "loads": [ { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "elapsed_seconds": 161.351567029953, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 1.0, "request_count": 64, "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.041293968822475, "ttft_max_ms": 86.51189290936045, "ttft_p50_ms": 86.51189290936001, "ttft_p95_ms": 86.51189290936045 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "elapsed_seconds": 9.191051244735718, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.038674169035575, "ttft_max_ms": 86.51189290936001, "ttft_p50_ms": 86.51189290935957, "ttft_p95_ms": 86.51189290936001 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "elapsed_seconds": 9.050537109375, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 15.535832140198398, "ttft_max_ms": 276.9276050512883, "ttft_p50_ms": 189.66908885537626, "ttft_p95_ms": 252.16908885537626 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "elapsed_seconds": 9.04306149482727, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", "score": { "feasible": true, "pass_rate": 0.984375, "passed": 63, "throughput_requests_per_second": 19.925437356037587, "ttft_max_ms": 1281.7127732631457, "ttft_p50_ms": 729.6185162106785, "ttft_p95_ms": 1219.2127732631457 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 8, "name": "tp4_mns8", "tp": 4 }, "elapsed_seconds": 9.14296841621399, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", "score": { "feasible": false, "pass_rate": 0.515625, "passed": 33, "throughput_requests_per_second": 20.34996647854275, "ttft_max_ms": 2191.8433254998167, "ttft_p50_ms": 1169.8561499618286, "ttft_p95_ms": 2106.0636248827373 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "loads": [ { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "elapsed_seconds": 9.14107608795166, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 1.0, "request_count": 64, "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.041293968822475, "ttft_max_ms": 86.51189290936045, "ttft_p50_ms": 86.51189290936001, "ttft_p95_ms": 86.51189290936045 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "elapsed_seconds": 9.139281272888184, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.038674169035575, "ttft_max_ms": 86.51189290936001, "ttft_p50_ms": 86.51189290935957, "ttft_p95_ms": 86.51189290936001 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "elapsed_seconds": 9.240875244140625, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 15.535832140198398, "ttft_max_ms": 276.9276050512883, "ttft_p50_ms": 189.66908885537626, "ttft_p95_ms": 252.16908885537626 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "elapsed_seconds": 9.090924739837646, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", "score": { "feasible": true, "pass_rate": 0.984375, "passed": 63, "throughput_requests_per_second": 19.925437356037587, "ttft_max_ms": 1281.7127732631457, "ttft_p50_ms": 729.6185162106785, "ttft_p95_ms": 1219.2127732631457 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 16, "name": "tp4_mns16", "tp": 4 }, "elapsed_seconds": 9.041213274002075, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", "score": { "feasible": false, "pass_rate": 0.515625, "passed": 33, "throughput_requests_per_second": 20.34996647854275, "ttft_max_ms": 2191.8433254998167, "ttft_p50_ms": 1169.8561499618286, "ttft_p95_ms": 2106.0636248827373 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "loads": [ { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "elapsed_seconds": 9.092233896255493, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 1.0, "request_count": 64, "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.041293968822475, "ttft_max_ms": 86.51189290936045, "ttft_p50_ms": 86.51189290936001, "ttft_p95_ms": 86.51189290936045 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "elapsed_seconds": 9.093050956726074, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.038674169035575, "ttft_max_ms": 86.51189290936001, "ttft_p50_ms": 86.51189290935957, "ttft_p95_ms": 86.51189290936001 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "elapsed_seconds": 9.242159605026245, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 15.535832140198398, "ttft_max_ms": 276.9276050512883, "ttft_p50_ms": 189.66908885537626, "ttft_p95_ms": 252.16908885537626 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "elapsed_seconds": 9.19024658203125, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", "score": { "feasible": true, "pass_rate": 0.984375, "passed": 63, "throughput_requests_per_second": 19.925437356037587, "ttft_max_ms": 1281.7127732631457, "ttft_p50_ms": 729.6185162106785, "ttft_p95_ms": 1219.2127732631457 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 32, "name": "tp4_mns32", "tp": 4 }, "elapsed_seconds": 9.18896198272705, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", "score": { "feasible": false, "pass_rate": 0.515625, "passed": 33, "throughput_requests_per_second": 20.34996647854275, "ttft_max_ms": 2191.8433254998167, "ttft_p50_ms": 1169.8561499618286, "ttft_p95_ms": 2106.0636248827373 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "loads": [ { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "elapsed_seconds": 9.088153839111328, "offered_request_rate": 4.0, "offered_request_rate_per_gpu": 1.0, "request_count": 64, "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 4.041293968822475, "ttft_max_ms": 86.51189290936045, "ttft_p50_ms": 86.51189290936001, "ttft_p95_ms": 86.51189290936045 }, "status": "completed", "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "elapsed_seconds": 9.048948764801025, "offered_request_rate": 8.0, "offered_request_rate_per_gpu": 2.0, "request_count": 64, "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 8.038674169035575, "ttft_max_ms": 86.51189290936001, "ttft_p50_ms": 86.51189290935957, "ttft_p95_ms": 86.51189290936001 }, "status": "completed", "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "elapsed_seconds": 9.04839539527893, "offered_request_rate": 16.0, "offered_request_rate_per_gpu": 4.0, "request_count": 64, "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", "score": { "feasible": true, "pass_rate": 1.0, "passed": 64, "throughput_requests_per_second": 15.535832140198398, "ttft_max_ms": 276.9276050512883, "ttft_p50_ms": 189.66908885537626, "ttft_p95_ms": 252.16908885537626 }, "status": "completed", "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "elapsed_seconds": 9.139516353607178, "offered_request_rate": 32.0, "offered_request_rate_per_gpu": 8.0, "request_count": 64, "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", "score": { "feasible": true, "pass_rate": 0.984375, "passed": 63, "throughput_requests_per_second": 19.925437356037587, "ttft_max_ms": 1281.7127732631457, "ttft_p50_ms": 729.6185162106785, "ttft_p95_ms": 1219.2127732631457 }, "status": "completed", "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" }, { "config": { "mns": 64, "name": "tp4_mns64", "tp": 4 }, "elapsed_seconds": 9.145836114883423, "offered_request_rate": 64.0, "offered_request_rate_per_gpu": 16.0, "request_count": 64, "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", "score": { "feasible": false, "pass_rate": 0.515625, "passed": 33, "throughput_requests_per_second": 20.34996647854275, "ttft_max_ms": 2191.8433254998167, "ttft_p50_ms": 1169.8561499618286, "ttft_p95_ms": 2106.0636248827373 }, "status": "completed", "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" } ] } ], "contract": { "arrival": "open_loop_uniform", "input_tokens": 2048, "output_tokens": 1, "prefix_caching": false, "rates": [ 4.0, 8.0, 16.0, 32.0, 64.0 ], "requests_per_anchor": 64, "target_pass_rate": 0.95, "ttft_slo_ms": 1256.0 }, "frontier": { "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", "git_status_short": " M frontier/cc_backend/backends/vidur_cc_backend.py\n M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n?? tests/unit/test_vidur_cc_large_payload.py\n", "source": "/tmp/replayserve-frontier-rs1b" }, "profiles": { "coverage": { "attention": { "1": { "exact_prefill_2048_rows": 1, "profile_batch_size": 1 }, "2": { "exact_prefill_2048_rows": 1, "profile_batch_size": 1 }, "4": { "exact_prefill_2048_rows": 1, "profile_batch_size": 1 } }, "manifest": { "attention_tp_coverage": [ 1, 2, 4 ], "environment_contract": { "dtype": "bfloat16", "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", "hardware": "NVIDIA H20", "model": "Qwen3-30B-A3B", "tensor_parallel_sizes": [ 1, 2, 4 ], "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", "vllm_version": "0.20.0" }, "inputs": { "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d", "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad", "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" }, "outputs": { "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", "attention.csv": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", "attention_true_mixed_fused.csv": "cccf76e6cb6a9359f188b07d4dc2942442568a6e8cf3ff3d77ab3d35a052cecb", "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", "moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" }, "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition", "projection_contract": { "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", "attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied", "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", "linear": "Frontier profiler using vLLM 0.20 CUDA operators", "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" }, "row_counts": { "allreduce": 24, "attention_frontier_compatible": 150, "attention_true_mixed_fused_diagnostic": 30, "linear": 36, "moe": 72 }, "schema_version": "frontier_qwen30_vllm020_frozen_profile.v3" } }, "root": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final", "sha256": { "attention": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", "manifest": "70aecb6d70f9e509e9816b1a0929163b9505760c2ac30bbd4cd8e51badfb7635", "moe": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" } }, "schema": "frontier-qwen30-prefill-surface-v1", "status": "frozen_before_real" }