diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json new file mode 100644 index 0000000..8671510 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json @@ -0,0 +1,790 @@ +{ + "cohort": [ + { + "input_bin": 0, + "input_tokens": 865, + "prompt_sha256": "c9e22fa0559b409b259bdb15ed6eec1bb36ecb8611ae367b2e122bc6303f56ae", + "sampling_u": 0.46149221772182225, + "source_arrival_s": 8.92599999999993, + "source_request_id": "250", + "source_row_index": 250 + }, + { + "input_bin": 4, + "input_tokens": 9154, + "prompt_sha256": "53b0a0af6cb1c0278183c61b8b6c026977b6cc46e07fca825b5bc8c5802b0e17", + "sampling_u": 0.8156596361943489, + "source_arrival_s": 12.708999999999833, + "source_request_id": "367", + "source_row_index": 367 + }, + { + "input_bin": 0, + "input_tokens": 718, + "prompt_sha256": "87d2846513ce3def0c4a7f9d3707f2eda02069f74535ba69dba6fb761bbfadf5", + "sampling_u": 0.9701010298706338, + "source_arrival_s": 25.190000000000055, + "source_request_id": "682", + "source_row_index": 682 + }, + { + "input_bin": 1, + "input_tokens": 1519, + "prompt_sha256": "7dbffc44084f1dfd9af43c2c40b5e669728a2c01b0a1e29f65ea17bd37e5bffc", + "sampling_u": 0.48115960917218603, + "source_arrival_s": 25.639999999999873, + "source_request_id": "690", + "source_row_index": 690 + }, + { + "input_bin": 1, + "input_tokens": 1968, + "prompt_sha256": "65e1015fbc28a8a2af5ffced6947478c4d68b39cd07a79c5e2fb937ecf9d64a7", + "sampling_u": 0.7394708415743882, + "source_arrival_s": 36.82400000000007, + "source_request_id": "1022", + "source_row_index": 1022 + }, + { + "input_bin": 0, + "input_tokens": 62, + "prompt_sha256": "6ba251e2a1653df7c13c70ed9aa3317af23f42d0ec14b14e46b1376b86f548b7", + "sampling_u": 0.5901445515787429, + "source_arrival_s": 39.96399999999994, + "source_request_id": "1098", + "source_row_index": 1098 + }, + { + "input_bin": 0, + "input_tokens": 702, + "prompt_sha256": "ec3dbd6ac6674268fb420b0e006f7edbbad38906f178faf4c41d5ac3aa7a7968", + "sampling_u": 0.4229843993649856, + "source_arrival_s": 60.55600000000004, + "source_request_id": "1571", + "source_row_index": 1571 + }, + { + "input_bin": 1, + "input_tokens": 1472, + "prompt_sha256": "1fc79d848cc4d3f92ee5a9ad6ecc28a3173737f77ec0b2eaca1b38026225c701", + "sampling_u": 0.38189150492188706, + "source_arrival_s": 69.46300000000019, + "source_request_id": "1792", + "source_row_index": 1792 + }, + { + "input_bin": 2, + "input_tokens": 2701, + "prompt_sha256": "084b29ad36178a06d93a1fdfbed02bc0718b42f43187d184bc06f70f03df4dfc", + "sampling_u": 0.8169083216699623, + "source_arrival_s": 69.79100000000017, + "source_request_id": "1799", + "source_row_index": 1799 + }, + { + "input_bin": 3, + "input_tokens": 7825, + "prompt_sha256": "044f618323ac49afa864bf6b62b1e06a9f3c351647a953cbe38b3173542caf51", + "sampling_u": 0.3751377474987089, + "source_arrival_s": 70.71099999999979, + "source_request_id": "1818", + "source_row_index": 1818 + }, + { + "input_bin": 0, + "input_tokens": 715, + "prompt_sha256": "e5f9aba41593f81170000f1299c71d0f09f52afc58addc2968fb862f8b740359", + "sampling_u": 0.7098663141194472, + "source_arrival_s": 75.24699999999984, + "source_request_id": "1935", + "source_row_index": 1935 + }, + { + "input_bin": 1, + "input_tokens": 1573, + "prompt_sha256": "d69852bb4dbc6b7653ed3a200e891190af5d052d20e57c2ff3abbccc6bf254aa", + "sampling_u": 0.6074460889792176, + "source_arrival_s": 84.5329999999999, + "source_request_id": "2176", + "source_row_index": 2176 + }, + { + "input_bin": 5, + "input_tokens": 29385, + "prompt_sha256": "86cb5ce843bad59ada0e7b9e1a60c1162534b037f5c1b5a7fadca5be26254047", + "sampling_u": 0.17186958800332738, + "source_arrival_s": 93.04399999999987, + "source_request_id": "2396", + "source_row_index": 2396 + }, + { + "input_bin": 1, + "input_tokens": 1939, + "prompt_sha256": "06bad3b6fc6db1c9227a8bb2d4ce349b31802e51e056b541858773debb0a388d", + "sampling_u": 0.29421530368234206, + "source_arrival_s": 98.4050000000002, + "source_request_id": "2505", + "source_row_index": 2505 + }, + { + "input_bin": 1, + "input_tokens": 1532, + "prompt_sha256": "9117e580cf5b8dcedb3c6c02318610c4780fbbf1c4413367db6f7eb60b73459b", + "sampling_u": 0.41616945652066173, + "source_arrival_s": 116.05200000000013, + "source_request_id": "2913", + "source_row_index": 2913 + }, + { + "input_bin": 0, + "input_tokens": 513, + "prompt_sha256": "6cf0a32f0a62d4a16f6abddf0bed8af587ba8f0aff335fd54c72f5772da4464a", + "sampling_u": 0.3613591846324354, + "source_arrival_s": 118.6239999999998, + "source_request_id": "2979", + "source_row_index": 2979 + }, + { + "input_bin": 1, + "input_tokens": 1508, + "prompt_sha256": "ad49c5bb6cf0ced603cc052215c18159d5c25927048ddb7efd6bf315bd84c937", + "sampling_u": 0.1650938738573754, + "source_arrival_s": 121.26600000000008, + "source_request_id": "3056", + "source_row_index": 3056 + }, + { + "input_bin": 0, + "input_tokens": 705, + "prompt_sha256": "c7b6f66c2ade7054abed59f4459bbe4ab4659a3d53654197d27ce26f650450e1", + "sampling_u": 0.8484961251249794, + "source_arrival_s": 130.1909999999998, + "source_request_id": "3301", + "source_row_index": 3301 + }, + { + "input_bin": 4, + "input_tokens": 8287, + "prompt_sha256": "e597785700703dc05da72b1045e734f326ed491276f2d9f45f663f03f1ccd509", + "sampling_u": 0.7289458530141959, + "source_arrival_s": 133.10300000000007, + "source_request_id": "3379", + "source_row_index": 3379 + }, + { + "input_bin": 1, + "input_tokens": 1756, + "prompt_sha256": "b9631ff1dfec71fbd9e3a084bccf30e40a058b74743ae9878e3dd4be77efdf83", + "sampling_u": 0.4082222424943361, + "source_arrival_s": 152.64800000000014, + "source_request_id": "3844", + "source_row_index": 3844 + }, + { + "input_bin": 0, + "input_tokens": 703, + "prompt_sha256": "a943002270d455f4e65022747cf40807044f47fb574089d95deadffed56fcf2e", + "sampling_u": 0.49508596140129646, + "source_arrival_s": 166.43499999999995, + "source_request_id": "4163", + "source_row_index": 4163 + }, + { + "input_bin": 1, + "input_tokens": 1478, + "prompt_sha256": "4b84065a6336bfb9ddd174099740689adfeb5ce294fe670885334934cf583d8b", + "sampling_u": 0.2476209159147146, + "source_arrival_s": 172.2800000000002, + "source_request_id": "4323", + "source_row_index": 4323 + }, + { + "input_bin": 0, + "input_tokens": 686, + "prompt_sha256": "b2154c41a891d72e417d3bb3d67920590349860655d93cb3ff478ad3da74e2f8", + "sampling_u": 0.7190432102580353, + "source_arrival_s": 193.67599999999993, + "source_request_id": "4838", + "source_row_index": 4838 + }, + { + "input_bin": 1, + "input_tokens": 1600, + "prompt_sha256": "9f4cafecff466decfe98d4a9544d05c6246853f24cfeb6e77c9e4d1a5cb6cb02", + "sampling_u": 0.71711230795038, + "source_arrival_s": 194.66400000000021, + "source_request_id": "4870", + "source_row_index": 4870 + }, + { + "input_bin": 2, + "input_tokens": 4000, + "prompt_sha256": "30c385ba8463097d52f838bbfa69f18a32c91b38878e15c11e6b69d6bf81abc7", + "sampling_u": 0.12696475188087025, + "source_arrival_s": 216.49299999999994, + "source_request_id": "5469", + "source_row_index": 5469 + }, + { + "input_bin": 0, + "input_tokens": 710, + "prompt_sha256": "807c8b29c95a0623c585ee7d15d82e72be5858f3683d3657ef3295e71ef6aa10", + "sampling_u": 0.32798737479046675, + "source_arrival_s": 223.39199999999983, + "source_request_id": "5648", + "source_row_index": 5648 + }, + { + "input_bin": 0, + "input_tokens": 727, + "prompt_sha256": "b872bc3174db8e4b17e182608dbb48f99196220dfd55096c2490484a6cadd595", + "sampling_u": 0.3644506007131907, + "source_arrival_s": 239.0999999999999, + "source_request_id": "6031", + "source_row_index": 6031 + }, + { + "input_bin": 4, + "input_tokens": 8786, + "prompt_sha256": "5757f2f9e6be7d4a29a79ad82408e51069044828f55b8c21b7216abbf73b0757", + "sampling_u": 0.12629800224853702, + "source_arrival_s": 260.4920000000002, + "source_request_id": "6604", + "source_row_index": 6604 + }, + { + "input_bin": 4, + "input_tokens": 9518, + "prompt_sha256": "730aee2b4ec388efbf4ca55cbfcbe31c8aa9636c8876b624b37b499d5dcb00f5", + "sampling_u": 0.29338926296423934, + "source_arrival_s": 260.91499999999996, + "source_request_id": "6611", + "source_row_index": 6611 + }, + { + "input_bin": 0, + "input_tokens": 796, + "prompt_sha256": "e0f7312dcc97af6236edac6e47a828f9e18e3e118023537d82f6cb540025191e", + "sampling_u": 0.47864598488768767, + "source_arrival_s": 274.52, + "source_request_id": "6905", + "source_row_index": 6905 + }, + { + "input_bin": 0, + "input_tokens": 228, + "prompt_sha256": "b85c14bcfaec63a99ebc382a7e20d32390a65999062e84c303441344832c3279", + "sampling_u": 0.8157098758872824, + "source_arrival_s": 274.6399999999999, + "source_request_id": "6911", + "source_row_index": 6911 + }, + { + "input_bin": 5, + "input_tokens": 18439, + "prompt_sha256": "8f7259429d8b3b6ab062c885d990fa2136e8ac061ce5b59169e5dbe7cdd5c055", + "sampling_u": 0.44219574639241527, + "source_arrival_s": 287.1959999999999, + "source_request_id": "7271", + "source_row_index": 7271 + }, + { + "input_bin": 0, + "input_tokens": 736, + "prompt_sha256": "1c56453602e159f1dfaae09b5b40785d0dbc176d9b27844362e342e3731c2032", + "sampling_u": 0.6889130017060773, + "source_arrival_s": 291.97400000000016, + "source_request_id": "7403", + "source_row_index": 7403 + }, + { + "input_bin": 4, + "input_tokens": 8557, + "prompt_sha256": "14346a1cc9d1382dd67eaf700f5b52798b77fc8a08bc1df71768d38d613e9e43", + "sampling_u": 0.3112632330750767, + "source_arrival_s": 292.692, + "source_request_id": "7422", + "source_row_index": 7422 + }, + { + "input_bin": 1, + "input_tokens": 1586, + "prompt_sha256": "4d24bf64da9142747d8ee8d87daf50f81309928c3d4792f7c345ecd9dd0c24ac", + "sampling_u": 0.23031083293934718, + "source_arrival_s": 304.69900000000007, + "source_request_id": "7741", + "source_row_index": 7741 + }, + { + "input_bin": 1, + "input_tokens": 1871, + "prompt_sha256": "5a4917df4ebb8a2e4e3dcb0f15c231cd46830af3dc63452ebf421b4433756159", + "sampling_u": 0.10577279384258167, + "source_arrival_s": 312.683, + "source_request_id": "7981", + "source_row_index": 7981 + }, + { + "input_bin": 0, + "input_tokens": 698, + "prompt_sha256": "610618e93c2f4d7bd0ca26b2611b61ffdbbbe092fe0ff3a5f7d7c91f52de7c74", + "sampling_u": 0.43535389643319383, + "source_arrival_s": 319.1579999999999, + "source_request_id": "8179", + "source_row_index": 8179 + }, + { + "input_bin": 0, + "input_tokens": 710, + "prompt_sha256": "3b397d03d7a7001ab8f3915b1b88ed8f8b859ad51769ac752ff35f567ccb1183", + "sampling_u": 0.6383291776591391, + "source_arrival_s": 321.4380000000001, + "source_request_id": "8224", + "source_row_index": 8224 + }, + { + "input_bin": 1, + "input_tokens": 1452, + "prompt_sha256": "671c93bcdd7a292d1a4b4dcce44c679369955fdb4d250d451170c63391216436", + "sampling_u": 0.12477813551209259, + "source_arrival_s": 337.8319999999999, + "source_request_id": "8612", + "source_row_index": 8612 + }, + { + "input_bin": 5, + "input_tokens": 24271, + "prompt_sha256": "a84a31f74c755d02154a7eaeec8bd71450c16813d8b7e713999ff52419f84ed6", + "sampling_u": 0.5538066537263959, + "source_arrival_s": 348.692, + "source_request_id": "8891", + "source_row_index": 8891 + }, + { + "input_bin": 3, + "input_tokens": 4607, + "prompt_sha256": "720cf238e668f4ae567ab909b2ec0185d222a14eb474ae2ef481141a885b3aad", + "sampling_u": 0.03539900309623126, + "source_arrival_s": 382.55499999999984, + "source_request_id": "9768", + "source_row_index": 9768 + }, + { + "input_bin": 0, + "input_tokens": 111, + "prompt_sha256": "8a12215d12ddcf36d1e9013ec0b4a433e4f27c5c2a774ead0f66c93bb31a8063", + "sampling_u": 0.9241501254905812, + "source_arrival_s": 402.8620000000001, + "source_request_id": "10305", + "source_row_index": 10305 + }, + { + "input_bin": 0, + "input_tokens": 714, + "prompt_sha256": "e1dd5099fb8565866691241945b15ee40a5ea8ad9045d10e1dd7774b668d9fa4", + "sampling_u": 0.4533435244095546, + "source_arrival_s": 405.0459999999998, + "source_request_id": "10383", + "source_row_index": 10383 + }, + { + "input_bin": 1, + "input_tokens": 1792, + "prompt_sha256": "9d94bba8c056a775a5d00ecfd6567e83030c99dc20f71ce53d787683ca2450e9", + "sampling_u": 0.8983379345538031, + "source_arrival_s": 405.08100000000013, + "source_request_id": "10385", + "source_row_index": 10385 + }, + { + "input_bin": 0, + "input_tokens": 697, + "prompt_sha256": "7a66c3cec2e1727fa78ed1a7dbe1d39d2c877cc2835c0b10479b396204749984", + "sampling_u": 0.43645445582603326, + "source_arrival_s": 413.16499999999996, + "source_request_id": "10618", + "source_row_index": 10618 + }, + { + "input_bin": 0, + "input_tokens": 709, + "prompt_sha256": "a71447eebb513ece438e0a81a0a9788fbbf34a71d9a66987e6f476cf76f0b7d5", + "sampling_u": 0.4730229506348141, + "source_arrival_s": 424.991, + "source_request_id": "10911", + "source_row_index": 10911 + }, + { + "input_bin": 0, + "input_tokens": 699, + "prompt_sha256": "e04aaa160224e82157f9b0c89180dc341e2c8eebbd518ea8d3327eb73ebfac1c", + "sampling_u": 0.5350250700640652, + "source_arrival_s": 430.51800000000003, + "source_request_id": "11058", + "source_row_index": 11058 + }, + { + "input_bin": 2, + "input_tokens": 3130, + "prompt_sha256": "ba93e2babcf7a0491ba97930f5b3bfb3cc2170dc3cd7b3ca3c3f9fe2905dc72e", + "sampling_u": 0.6597844956490088, + "source_arrival_s": 434.3449999999998, + "source_request_id": "11166", + "source_row_index": 11166 + }, + { + "input_bin": 0, + "input_tokens": 711, + "prompt_sha256": "80c05739d748df64d97f92742f28e384365b26f4f5f1e999666ea2f7f187fd16", + "sampling_u": 0.7240395487416784, + "source_arrival_s": 441.3699999999999, + "source_request_id": "11373", + "source_row_index": 11373 + }, + { + "input_bin": 1, + "input_tokens": 1536, + "prompt_sha256": "ee1209838aca67d2392bb9156b74d69de22d166d97c88b56501ca9cfcc70578e", + "sampling_u": 0.9806533325226275, + "source_arrival_s": 463.8670000000002, + "source_request_id": "11911", + "source_row_index": 11911 + }, + { + "input_bin": 1, + "input_tokens": 1580, + "prompt_sha256": "a4e5fdb5d6119e9207384fd3d32ac027469b3ecca278738c7078026551999786", + "sampling_u": 0.8508070546796023, + "source_arrival_s": 471.08899999999994, + "source_request_id": "12100", + "source_row_index": 12100 + }, + { + "input_bin": 1, + "input_tokens": 1466, + "prompt_sha256": "09a6942491467ee3597eef41c4f36de46bd88b0533af8c1084bb9e15cb150832", + "sampling_u": 0.03536418626599094, + "source_arrival_s": 488.5859999999998, + "source_request_id": "12587", + "source_row_index": 12587 + }, + { + "input_bin": 0, + "input_tokens": 750, + "prompt_sha256": "6a53081dfa161f2b1e50e03885b0d2868c7390f91130d0a1cd95b4b75cdb1126", + "sampling_u": 0.4344415561103455, + "source_arrival_s": 488.6289999999999, + "source_request_id": "12588", + "source_row_index": 12588 + }, + { + "input_bin": 0, + "input_tokens": 736, + "prompt_sha256": "3514d193ff83b6d6589d52f24072b0159fa00d7b44ca8c6ec269515db4cfba89", + "sampling_u": 0.7257534252552246, + "source_arrival_s": 497.2449999999999, + "source_request_id": "12824", + "source_row_index": 12824 + }, + { + "input_bin": 0, + "input_tokens": 90, + "prompt_sha256": "915b3aee8d1288262c7f363f9035af19236e45540b748a9ced17c2d23f428f34", + "sampling_u": 0.7669962160527245, + "source_arrival_s": 520.1750000000002, + "source_request_id": "13350", + "source_row_index": 13350 + }, + { + "input_bin": 0, + "input_tokens": 697, + "prompt_sha256": "af93c55d3eb4f5161eb69410e73039e1d500a92190bff884e1b4040e491c8ed5", + "sampling_u": 0.594626944330893, + "source_arrival_s": 525.469, + "source_request_id": "13491", + "source_row_index": 13491 + }, + { + "input_bin": 1, + "input_tokens": 1470, + "prompt_sha256": "dd86d891c322f43310d4207c02f27a7d5e76885c420793164c76020f7ff166c4", + "sampling_u": 0.9612563305576178, + "source_arrival_s": 526.116, + "source_request_id": "13510", + "source_row_index": 13510 + }, + { + "input_bin": 1, + "input_tokens": 1592, + "prompt_sha256": "2ab08103a5ec0e01aea2f74f9215c52222ef8722a2fb9327bd9187a644df1c4b", + "sampling_u": 0.7977149220573079, + "source_arrival_s": 529.1809999999996, + "source_request_id": "13612", + "source_row_index": 13612 + }, + { + "input_bin": 5, + "input_tokens": 26776, + "prompt_sha256": "d09d69f9110f28c69d0767bdd501a8bdba167775ae1c6e6c53d15cf0fdcf5f6e", + "sampling_u": 0.9888861232028356, + "source_arrival_s": 546.9939999999997, + "source_request_id": "14026", + "source_row_index": 14026 + }, + { + "input_bin": 1, + "input_tokens": 1589, + "prompt_sha256": "973120a2be961f8f44523988a44f59b2071e82081a292cf37f10fb72d2874535", + "sampling_u": 0.8325378218001509, + "source_arrival_s": 549.6970000000001, + "source_request_id": "14094", + "source_row_index": 14094 + }, + { + "input_bin": 1, + "input_tokens": 1550, + "prompt_sha256": "8d899d80a6b9c47f414f6d65ed1638b5fc3bc782eaa19b2592878f35ff5484e7", + "sampling_u": 0.7285184471546545, + "source_arrival_s": 556.8310000000001, + "source_request_id": "14316", + "source_row_index": 14316 + }, + { + "input_bin": 2, + "input_tokens": 2501, + "prompt_sha256": "4a88c92c72a1fd9a653a7dbbd169eff30774f1b95cb5482f60c45532923d9f47", + "sampling_u": 0.38599257949920635, + "source_arrival_s": 558.3209999999999, + "source_request_id": "14359", + "source_row_index": 14359 + }, + { + "input_bin": 0, + "input_tokens": 741, + "prompt_sha256": "9c2341411206e2d28d6479ef20d5ce9e90daac574e72c4066130e2601731624c", + "sampling_u": 0.5930037066322071, + "source_arrival_s": 583.5829999999996, + "source_request_id": "15069", + "source_row_index": 15069 + }, + { + "input_bin": 3, + "input_tokens": 6044, + "prompt_sha256": "614a0d743b4cec35549b490110bc71d8fc3a22cd07c68de7281fc98270f81ee8", + "sampling_u": 0.4726761733329724, + "source_arrival_s": 594.8950000000004, + "source_request_id": "15377", + "source_row_index": 15377 + } + ], + "created_unix_s": 1784139549.1404486, + "load_contract": { + "binary_search": false, + "completion_tokens_override": 1, + "monotonicity_assumed": false, + "offered_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "only_mutated_variable": "uniform_arrival_timeline_scale", + "request_count_per_point": 64, + "target_pass_rate": 0.95 + }, + "primary_slo": "linear_8k_primary", + "rates": { + "r_0p150": { + "duration_s": 426.6666666666667, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.15, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p150.csv", + "request_count": 64, + "request_rate_recomputed": 0.15, + "sha256": "f35ce4424c17b2ee16144bb59011e467a07d3f57895716165fcdc73ba19839d9", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p200": { + "duration_s": 320.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.2, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p200.csv", + "request_count": 64, + "request_rate_recomputed": 0.2, + "sha256": "b81a48706a5c41222477aef8da357ae14c7b0bc19d35ea5ecc0c12b86d32590c", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p250": { + "duration_s": 256.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.25, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p250.csv", + "request_count": 64, + "request_rate_recomputed": 0.25, + "sha256": "721d622abb5d0a9080d6577063af910906b4e9928b402f48804ee55b25f108fc", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p300": { + "duration_s": 213.33333333333334, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.3, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p300.csv", + "request_count": 64, + "request_rate_recomputed": 0.3, + "sha256": "a5c8a7b93f314bf4365763fc0157597c582229993bb79b41b697ef3247495e49", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p350": { + "duration_s": 182.85714285714286, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.35, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p350.csv", + "request_count": 64, + "request_rate_recomputed": 0.35, + "sha256": "ec9debce28b76981f4f5363f0c77faa963c550a5d3ecbed327a304dc0d495242", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p400": { + "duration_s": 160.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.4, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p400.csv", + "request_count": 64, + "request_rate_recomputed": 0.4, + "sha256": "aea1cae38b09ddb0ad1a7d6da03bd6ad76afdaee3b2879b7cb65eba9b3ef5bab", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p450": { + "duration_s": 142.22222222222223, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.45, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p450.csv", + "request_count": 64, + "request_rate_recomputed": 0.44999999999999996, + "sha256": "672197deca834d9b9c5aea963448805a03e86495721d5eb270ccec2416faada2", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p500": { + "duration_s": 128.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.5, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p500.csv", + "request_count": 64, + "request_rate_recomputed": 0.5, + "sha256": "7cd2d16904e0fafb26a67718c4f2d7b6752934864219ab96813b8f62b47019d4", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p750": { + "duration_s": 85.33333333333333, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.75, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p750.csv", + "request_count": 64, + "request_rate_recomputed": 0.75, + "sha256": "e8b00f14b6c85ac8bd1c8fe3b83797f5c61903d70d8bbf6acfe9a1bd384db443", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_1p000": { + "duration_s": 64.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 1.0, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_1p000.csv", + "request_count": 64, + "request_rate_recomputed": 1.0, + "sha256": "7071e0247414af9214a1bf06f05a4865965568ba812bbae6713648e994a40a34", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_1p500": { + "duration_s": 42.666666666666664, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 1.5, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_1p500.csv", + "request_count": 64, + "request_rate_recomputed": 1.5, + "sha256": "86b03ef1fe87ad3c589f1585a279cd3136975009316e36e0910fe8340b972eee", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + } + }, + "schema": "qwen235b-prefill-fixed-cohort-protocol-v1", + "selection": { + "cohort_bin_quotas": [ + 27, + 21, + 4, + 3, + 5, + 4 + ], + "cohort_size": 64, + "cohort_source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5", + "input_bin_edges": [ + 0, + 1024, + 2048, + 4096, + 8192, + 16384, + 32769 + ], + "method": "length_stratified_smallest_sha256", + "seed": 2026071501, + "source_arrival_span_s": 585.9690000000005, + "source_bin_counts": [ + 6549, + 5111, + 916, + 743, + 1194, + 896 + ], + "stable_order": [ + "source_arrival_s", + "source_row_index" + ] + }, + "slo_variants": { + "legacy_step_1s_2s": { + "buckets": [ + { + "max_input_tokens": 8191, + "threshold_ms": 1000.0 + }, + { + "threshold_ms": 2000.0 + } + ], + "description": "Original strict 1 s/2 s step SLO (null-capacity sensitivity)", + "kind": "step_ms" + }, + "linear_10k": { + "description": "TTFT <= 1000 ms + input_tokens / 10000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.1 + }, + "linear_6k": { + "description": "TTFT <= 1000 ms + input_tokens / 6000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.16666666666666666 + }, + "linear_8k_primary": { + "description": "TTFT <= 1000 ms + input_tokens / 8000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.125 + } + }, + "source": { + "eligible_request_count": 15409, + "path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/thinking_w20260327_1000.jsonl", + "sha256": "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2" + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md new file mode 100644 index 0000000..dfe9855 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md @@ -0,0 +1,204 @@ +# Qwen3-235B FP8:Frontier 与 community vLLM config rank 对比 + +## 结论 + +在本实验限定的 Qwen3-235B-A22B-FP8、H20、prefill-only workload 上,经过 best-effort 补齐 FP8/MoE serving path 后,Frontier **能够选对满足 SLO 的 per-GPU throughput 最优配置**:`TP=4, MBT=16384`,`MNS=64/128` 并列。细化网格上的 Spearman rank correlation 为 `0.9487`,20 个可比较非 tie config pair 全部同序,选择 regret 为 `0`。 + +但这不等于 Frontier 已精确模拟系统。原始粗网格给出 56/56 load labels 正确、Spearman=1 的表象;加入 decision-boundary rates 后,Frontier 在 33 个被测试边界点中产生 6 个 false-infeasible,均为两个 MNS 配置上可复现的稳定误差。它系统性地把三个 action family 的最大经验 SLO throughput 低估一个 `0.05 req/s` 档位: + +| TP | MBT | Frontier capacity | real capacity | real v2→v3 transition | 结论 | +|---:|---:|---:|---:|---|---| +| 4 | 8192 | 0.15 | 0.20 | `0.20: 61→61`, `0.25: 60→60` | 稳定低估 0.05 | +| 4 | 16384 | 0.25 | 0.30 | `0.30: 61→61`, `0.35: 59→59` | 稳定低估 0.05 | +| 8 | 8192 | 0.40 | 0.45 | `0.45: 61→61`, `0.50: 60→60` | 稳定低估 0.05 | +| 8 | 16384 | 0.45 | 0.45 | v2: `0.45: 61`, `0.50: 60` | 边界正确 | + +表中 pass count 分母均为 64,经验可行门槛为 61。每一行的两个 MNS 结果一致;v3 对前三行的两个 MNS 都做了 fresh-server 复测。 + +因此,对这个 case 最准确的判断是: + +- Frontier 有较好的 **decision/ordinal fidelity**,不能再用“它找不到最优 config”作为该 case 的研究 premise。 +- Frontier 没有达到 **capacity calibration fidelity**,也没有完全学对 action effect;它适合筛选,但不能在 SLO knee 附近代替真机 capacity certification。 +- 粗网格上的“完全匹配”并不足以证明 simulator solve 了问题;它会同时掩盖绝对容量误差和错误的 action differential。 + +## 固定实验口径 + +| 项目 | 固定值 | +|---|---| +| model | `models/Qwen/Qwen3-235B-A22B-FP8` | +| real runtime | community vLLM 0.10.2 | +| hardware | dash0,8×NVIDIA H20 | +| workload | prefill-only,raw completion,output tokens=1,prefix cache disabled | +| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}` | +| cohort | 64 个固定 prompt;所有 config/rate 的 request ID、顺序、输入长度完全相同 | +| input length | p50=1478,p95=18439,max=29385,mean=3511.55 tokens | +| rates | 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75, 1.00, 1.50 req/s | +| primary SLO | `TTFT ≤ 1000 ms + input_tokens / 8000 tokens/s`,至少 95% requests 通过 | +| capacity | 所有 64 requests 完整结束后,最大被测试可行 offered rate;不做 early stop/二分单调假设 | +| primary objective | capacity / TP,即 per-GPU request throughput | + +cohort 只通过 uniform arrival scaling 改变 load,不随 rate 重新抽样。该设计消除了 workload composition confound,但 64-request 固定 cohort 的结果仍是经验容量,不代表总体 workload distribution 的置信界。 + +## Frontier best-effort 公平性 + +这里比较的不是未经修改的 stock Frontier。为了避免“故意把 baseline 调差”的质疑,我们在真机 ranking 冻结前完成并冻结了以下补齐: + +- 使用 community vLLM 0.10.2 在 H20 上得到的真实 FP8 operator profiles;quantization contract 为 FP8 W8A8 dynamic activation、BF16 output/accumulation,而不是用 BF16 compute profile 冒充 FP8。 +- 补齐 Qwen MoE community serving path、MoE plan 与 serving entrypoint。 +- 按真实 expert route 构造 EP local lanes,并以最慢 lane 作为同步 critical path。 +- 修正 TP/EP-aware profile cache key,避免不同 topology 错用缓存结果。 +- 使用 batched lane prediction;与逐 lane exact path 做 byte-identical equivalence 后才用于完整 sweep,模拟运行加速约 2.57×。 +- KV block capacity 使用真机测量值:TP4=26101,TP8=62351。 +- Frontier refined 8×11=88 个结果在真机运行前冻结;旧 rates 的 56 个结果与先前 freeze byte-identical。没有用真机 v2/v3 结果回调 simulator 参数。 + +相关实现与审计入口包括: + +- `fixed_cohort_rank.py` +- `frontier_fp8_profile_contract.patch` +- `frontier_moe_serving_path.patch` +- `frontier_moe_ep_prefill_exact_cache.patch` +- `frontier_moe_ep_prefill_use_batched_lanes.patch` +- `frontier_moe_ep_prefill_batched_lane_predict.patch` + +这说明结果反映的是“给 Frontier 当前抽象最大的合理帮助以后,它能做到什么”,不是 stock baseline 的缺功能测试。 + +## 三阶段实验结果 + +### v1:blind 全表面、粗网格 + +Frontier 先冻结,再运行 community vLLM 的 8 configs × 7 rates: + +- 56/56 config-load primary feasibility labels 正确。 +- raw capacity 与 per-GPU efficiency 的 top set 均完全匹配。 +- Spearman=1,20/20 非 tie pair 同序,top-1 regret=0。 +- 但 3584 个 request 的 TTFT MAE 为 124.20 ms,MAPE 为 20.98%,平均 `predicted-real=+93.20 ms`。 +- request-level SLO label accuracy 为 96.82%;Frontier/real failure-set Jaccard 仅 0.7569(103 false fail、11 false pass)。 + +因此,aggregate capacity 完全匹配时,具体失败请求和 request latency mechanism 已明显不匹配。 + +### v2:反向 config/rate 顺序,加入细化边界 + +v2 只重跑预注册 SLO transition 及中间 refinement rates,共 8 configs、33 loads、2112 requests: + +- 0 server/request failure,0 early stop,0 monotonicity violation;所有 trace hash 匹配 refined protocol。 +- per-GPU top set 仍精确匹配:`TP4, MBT16384, MNS64/128`。 +- Frontier/real per-GPU capacity Spearman=0.9487,20/20 非 tie pair 同序,regret=0。 +- 33 个边界点中 27 个 feasibility label 正确;6 个错误全部是 false-infeasible,没有 false-feasible。 +- pass-rate MAE=1.278 percentage points,平均 `Frontier-real=-1.278 pp`。 +- 2112 个 request 的 TTFT MAE=83.15 ms,MAPE=16.34%,平均 `predicted-real=+53.02 ms`。 +- request SLO label accuracy=98.34%,failure-set Jaccard=0.7535。 + +v1 与 v2 的 19 个共同 config-load 点包含 1216 个相同 requests: + +- 真机 trial-to-trial TTFT MAE=14.05 ms,absolute-error p95=57.94 ms,平均 v2-v1=+1.75 ms。 +- primary request label 只有 1/1216 flip;19/19 load labels 和 8/8 原始网格 capacity 全部稳定。 +- Frontier 有 23 个 request label 在两次真机 trial 中都稳定预测错误。 + +这排除了 config order 和普通 runtime noise 是主要 residual 来源。 + +### v3:只复测 v2 的全部争议 transition + +v3 的选择规则是 v2 中所有 Frontier/real primary capacity 不同的 config;它是 data-dependent replication,不作为新的 blind primary result。共 6 configs、12 loads、768 requests: + +- 0 failure/early-stop/monotonicity violation,trace hash 全部匹配。 +- v2 与 v3 的 768/768 primary request labels 完全一致,failure-set Jaccard=1.0。 +- 12/12 config-load labels、6/6 capacity boundaries 完全一致。 +- 真机 trial-to-trial TTFT MAE=14.98 ms,absolute-error p95=62.52 ms,平均 v3-v2=+1.55 ms。 +- Frontier 在该争议集合上仍有 6/12 false-infeasible;18 个 request labels 在 v2/v3 都稳定预测错误。 + +这证明细化边界误差不是一次随机跨阈值。 + +## 系统层面的理解 + +### 1. 绝对误差与 action differential 必须分开 + +如果只是一个全局 latency scale 偏差,所有 topology 下 MBT 的相对收益应基本保持。数据并非如此: + +- TP4:Frontier 预测 MBT `8192→16384` 的 capacity 增益为 `0.15→0.25`,即 +0.10 req/s;真机为 `0.20→0.30`,同样 +0.10。 +- TP8:Frontier 预测 `0.40→0.45`,即 +0.05;真机为 `0.45→0.45`,实际增益为 0。 + +也就是说,Frontier 在 TP4 学对了 MBT differential、错了 absolute offset;在 TP8 则连 differential 都错了。该误差依赖 TP/EP topology,不能靠一个全局 calibration factor 修复。这正是 execution configuration 与 scheduler state/action effect 相互耦合的证据。 + +### 2. 决策正确不代表 simulator 对系统有正确解释 + +per-GPU objective 下,TP4/MBT16384 与其他配置有足够大的 margin,所以即便 capacity 每类偏差 0.05,最优 config 仍不变。raw single-replica objective 下,Frontier 只把 TP8/MBT16384 列为 top set,而真机四个 TP8 config 全部在 0.45 并列;虽然从 Frontier top set 中任选仍是 zero regret,但它虚构了 TP8 上 MBT 的收益。 + +因此需要同时报告: + +1. selected-config regret; +2. rank/order fidelity; +3. capacity error; +4. action differential error; +5. request/failure-set fidelity。 + +只报其中任意一层都会得到偏颇结论。 + +### 3. residual 是状态/长度相关尾部误差,不是 profile 全局比例错误 + +v2 中 Frontier 的平均 signed TTFT error 随输入长度显著变化: + +| input tokens | mean `predicted-real` | MAE | +|---|---:|---:| +| [0,1024) | -3.61 ms | 40.74 ms | +| [1024,2048) | +44.45 ms | 57.63 ms | +| [2048,4096) | +91.21 ms | 109.73 ms | +| [4096,8192) | +138.80 ms | 140.80 ms | +| [8192,16384) | +237.37 ms | 237.37 ms | +| [16384,32769) | +147.36 ms | 240.89 ms | + +短请求甚至略偏乐观,中长请求明显偏保守;与此同时 aggregate load label 只由 1–2 个请求是否跨 SLO 阈值决定。这更像 batch composition、critical-lane/EP synchronization 和 queueing state 共同造成的 residual,而不是 operator profile 全局慢了固定百分比。 + +### 4. MNS 在该 workload 上是非活跃维度,但不能外推 + +在所有共同测试点中,MNS=64/128 的 pass counts 基本逐点相同,Frontier 也预测 tie。对这个 prefill-only、固定 cohort、当前 concurrency 范围,MBT/token budget 和 topology 是活跃因素,MNS 没有形成独立瓶颈。但 decode/mixed workload 的 sequence residency、KV pressure 和 iteration cadence 不同,不能据此宣称 MNS 普遍无效。 + +## 对研究方向的判断 + +这个 case 不支持以下论文 premise: + +> Frontier 类 simulator 无法为 Qwen3-235B FP8 选择最优配置,所以必须由一个新 tuner 替代。 + +在公平补齐 FP8/MoE path 后,它已经能以 zero regret 选出该 workload 的 per-GPU 最优配置。继续用这一 case 证明“simulator 排序失败”会被 reviewer 用本组数据直接反驳。 + +本 case 真正揭示、且更有 systems 内容的问题是: + +> 一个 simulator 可以在 coarse grid 上拥有完美 rank,并在最终 objective 上 zero regret,同时仍错误表达 topology-dependent action differential 和 SLO knee;我们如何判断一个模拟决策何时足够可信,何时必须回到真机? + +可继续发展的研究问题包括: + +- decision-aware uncertainty:不追求每条 TTFT 精确,而是给 config ordering/capacity boundary 提供可校验置信区间。 +- counterexample-guided measurement:只在模拟 action margin 小或 differential 不稳定的 boundary/state 上调用真机,而不是全表面 sweep。 +- topology-aware residual model:显式建模 TP/EP、batch composition、critical lane 与输入长度共同造成的尾部偏差,避免全局比例校准。 +- distinguish safe screening from certification:Frontier 当前在该 case 是保守筛选器(0 false-feasible),但不是精确 capacity certifier。 + +下一步要判断这是否足以构成更一般的 research premise,应转向已计划的 Qwen3-235B decode-only 和 mixed prefill/decode case。它们会激活 KV state、replica residency、iteration cadence 和 communication overlap;如果只有这些 case 出现稳定排序/regret 错误,研究问题就应围绕 simulator 在 stateful execution/scheduling coupling 下的决策失效,而不是泛化地宣称 simulator 不工作。 + +## 证据与复现 + +本目录保存紧凑 comparison、trial-stability 与 manifest;完整 request-level freeze 保留在 dash0: + +`/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1` + +关键 artifact SHA256: + +| artifact | SHA256 | +|---|---| +| refined protocol manifest | `695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb` | +| Frontier refined freeze | `318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336` | +| v1 blind comparison | `fc87c54179b891e2148a738dbdfeed328595fa32198f2248c892c890625c9fe2` | +| v2 run manifest | `cf1a722043f42ff8f435e3521d0b2b2892a8ea0922e1e4870ec10fba26c0f998` | +| v2 community freeze | `5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2` | +| v2 comparison | `d21f2dd846c4d8361668f312f0c063f2e8f949ec7a9d3b19100c813b3da3b586` | +| v1-v2 stability | `a522189618ad26df68f18d2d9b67c38b1fe59c5c8f13af633eced0b092f8e06d` | +| v3 run manifest | `0f8908671b8903e516f0dc8e68f556483a15ff7ec51f7cfbc18a9c74b3a5ba61` | +| v3 community freeze | `4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34` | +| v3 comparison | `e04f40ead995e007d64b2f6f0f2731999204907770f05b4c2d037d224e49c573` | +| v2-v3 stability | `7be2ff27dd0393c99f0512bd6a0b7b35eb404ee0bfd22bd3ec33a4b8ce14748b` | + +## 限制 + +- 只测试了一个模型、H20、community vLLM 0.10.2 和 prefill-only workload。 +- 固定 64-request cohort 适合 simulator-vs-real paired comparison,但不足以给总体 workload distribution 的 95% SLO capacity 做统计认证。 +- v3 是在观察 v2 后选择全部争议点的 replication,不是 blind discovery dataset。 +- operator profiles、KV capacity 和 MoE route contract 本身使用了真机信息;本结果评估的是 best-effort simulator 的决策能力与剩余误差,不是 zero-measurement transfer。 +- 没有测试 decode、mixed phase、prefix caching、multi-replica placement、异构 GPU 或跨节点通信。 diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json new file mode 100644 index 0000000..001eafa --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json @@ -0,0 +1,6384 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "created_unix_s": 1784139490.8308787, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/frontier-v3-batched-par4/frontier_ranking_frozen.json", + "sha256": "9a971f25cc97975845968cc776241c7d44a49dee2ec2f015f227bcff82e66971" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.041294642857142856, + "matched_config_load_points": 56, + "max_absolute_error": 0.15625, + "signed_error_mean": -0.03962053571428571 + }, + "linear_10k": { + "false_feasible_points": 2, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 0.9642857142857143, + "mae": 0.033482142857142856, + "matched_config_load_points": 56, + "max_absolute_error": 0.140625, + "signed_error_mean": -0.029575892857142856 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 1, + "feasibility_label_accuracy": 0.9821428571428571, + "mae": 0.022600446428571428, + "matched_config_load_points": 56, + "max_absolute_error": 0.109375, + "signed_error_mean": -0.020926339285714284 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.026227678571428572, + "matched_config_load_points": 56, + "max_absolute_error": 0.125, + "signed_error_mean": -0.025669642857142856 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 1512, + "mae_ms": 80.67140471814413, + "signed_error_ms_mean": 40.081426915862 + }, + "[1024,2048)": { + "count": 1176, + "mae_ms": 113.2322821016931, + "signed_error_ms_mean": 95.35554067785822 + }, + "[16384,32769)": { + "count": 224, + "mae_ms": 248.44144323616518, + "signed_error_ms_mean": 151.67347724381298 + }, + "[2048,4096)": { + "count": 224, + "mae_ms": 146.4061790530529, + "signed_error_ms_mean": 116.84675686840379 + }, + "[4096,8192)": { + "count": 168, + "mae_ms": 150.66978579178092, + "signed_error_ms_mean": 148.27281627513304 + }, + "[8192,16384)": { + "count": 280, + "mae_ms": 272.270894330094, + "signed_error_ms_mean": 272.270894330094 + } + }, + "mae_ms": 124.19937765168956, + "mape": 0.20982454623810715, + "matched_successful_requests": 3584, + "signed_error_ms_mean": 93.2018552795005 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "1799", + "3379", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1935", + "4163", + "4323", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "1799", + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1935", + "4163", + "4323", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "1935", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "15377", + "1799", + "3379", + "3844", + "6611", + "6905", + "6911", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.84375, + "matched_requests": 64, + "matching_labels": 54, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "1935", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "15377", + "1799", + "3379", + "3844", + "6611", + "6905", + "6911", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.84375, + "matched_requests": 64, + "matching_labels": 54, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 581, + "failure_set_jaccard": 0.7926330150068213, + "failure_set_union": 733, + "false_slo_fail_requests": 147, + "false_slo_pass_requests": 5, + "frontier_failure_requests": 728, + "label_accuracy": 0.9575892857142857, + "matched_requests": 3584, + "matching_labels": 3432, + "real_failure_requests": 586 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098", + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "6604", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "6604", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6604", + "6905", + "6911", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "6604", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "6604", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6604", + "6905", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "3379", + "7271", + "7981", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "2396", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "15069", + "2396", + "3379", + "6611", + "7271", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.859375, + "matched_requests": 64, + "matching_labels": 55, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "3379", + "7271", + "7981", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "2396", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "15069", + "2396", + "3379", + "6611", + "7271", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.859375, + "matched_requests": 64, + "matching_labels": 55, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 415, + "failure_set_jaccard": 0.7167530224525043, + "failure_set_union": 579, + "false_slo_fail_requests": 135, + "false_slo_pass_requests": 29, + "frontier_failure_requests": 550, + "label_accuracy": 0.9542410714285714, + "matched_requests": 3584, + "matching_labels": 3420, + "real_failure_requests": 444 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6905", + "6911", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6905", + "6911", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "2505" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2913", + "2979", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "2505" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 306, + "failure_set_jaccard": 0.7906976744186046, + "failure_set_union": 387, + "false_slo_fail_requests": 78, + "false_slo_pass_requests": 3, + "frontier_failure_requests": 384, + "label_accuracy": 0.9773995535714286, + "matched_requests": 3584, + "matching_labels": 3503, + "real_failure_requests": 309 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14026", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7741" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "7981" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "6611", + "7271", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6611", + "6905", + "6911", + "7271", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14026", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7741" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "7981" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "6611", + "7271", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "7271", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.875, + "matched_requests": 64, + "matching_labels": 56, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 355, + "failure_set_jaccard": 0.7569296375266524, + "failure_set_union": 469, + "false_slo_fail_requests": 103, + "false_slo_pass_requests": 11, + "frontier_failure_requests": 458, + "label_accuracy": 0.9681919642857143, + "matched_requests": 3584, + "matching_labels": 3470, + "real_failure_requests": 366 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.84375, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.84375, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.84375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625, + 0.859375, + 0.84375, + 0.78125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625, + 0.859375, + 0.84375, + 0.78125, + 0.75, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.8125, + 0.734375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.8125, + 0.734375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.859375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.859375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.765625, + 0.671875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.75, + 0.625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.875, + 0.84375, + 0.8125, + 0.765625, + 0.671875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.75, + 0.609375 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 12, + "real_tied_pairs": 13 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt16384": 0.0625, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192", + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": 0.7745966692414834, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.5, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.921875, + 0.84375 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 12, + "real_tied_pairs": 9 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt16384": 0.5, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": 0.9561828874675149, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.30000000000000004, + "top1_set_intersection": [ + "tp8_mns128_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.5, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.30000000000000004, + "top1_set_intersection": [ + "tp8_mns128_mbt16384" + ], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.890625, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.890625, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.78125, + 0.6875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.78125, + 0.6875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.90625, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.90625, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.6875 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.703125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json new file mode 100644 index 0000000..b64a7d5 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json @@ -0,0 +1,1038 @@ +{ + "created_unix_s": 1784150483.6645162, + "inputs": { + "first_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "repeat_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + } + }, + "primary_original_grid_capacity_stability": { + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "records": [ + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt16384", + "first_capacity": 0.25, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt8192", + "first_capacity": 0.15, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt16384", + "first_capacity": 0.25, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt8192", + "first_capacity": 0.15, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt16384", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt8192", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt16384", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt8192", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "stable_configs": 8, + "total_configs": 8 + }, + "real_trial_ttft_ms": { + "absolute_error_p95": 57.944677013438195, + "mae": 14.052099088554602, + "paired_successful_requests": 1216, + "repeat_minus_first_mean": 1.7543344055363712 + }, + "schema": "community-qwen235b-trial-stability-v1", + "shared_config_load_points": 19, + "shared_requests": 1216, + "variants": { + "legacy_step_1s_2s": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 52, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 52, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + } + ], + "frontier_correct_first": 1177, + "frontier_correct_repeat": 1178, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9679276315789473, + "frontier_request_label_accuracy_repeat": 0.96875, + "frontier_wrong_both_stable_real_label": 38, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9925373134328358, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_10k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1173, + "frontier_correct_repeat": 1174, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9646381578947368, + "frontier_request_label_accuracy_repeat": 0.9654605263157895, + "frontier_wrong_both_stable_real_label": 42, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9891304347826086, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_6k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 63, + "frontier_feasible": true, + "frontier_passed_request_count": 63, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 63 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 63, + "frontier_feasible": true, + "frontier_passed_request_count": 63, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 63 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1206, + "frontier_correct_repeat": 1208, + "frontier_load_label_accuracy_first": 0.9473684210526315, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9917763157894737, + "frontier_request_label_accuracy_repeat": 0.993421052631579, + "frontier_wrong_both_stable_real_label": 8, + "matching_real_trial_labels": 1214, + "real_trial_failure_set_jaccard": 0.9655172413793104, + "real_trial_label_flips": 2, + "real_trial_load_label_accuracy": 0.9473684210526315, + "real_trial_request_label_accuracy": 0.9983552631578947, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_8k_primary": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1192, + "frontier_correct_repeat": 1193, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9802631578947368, + "frontier_request_label_accuracy_repeat": 0.9810855263157895, + "frontier_wrong_both_stable_real_label": 23, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9852941176470589, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json new file mode 100644 index 0000000..9574186 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json @@ -0,0 +1,5248 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "created_unix_s": 1784150358.538842, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.028409090909090908, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.026515151515151516 + }, + "linear_10k": { + "false_feasible_points": 0, + "false_infeasible_points": 8, + "feasibility_label_accuracy": 0.7575757575757576, + "mae": 0.022727272727272728, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.020833333333333332 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.9393939393939394, + "mae": 0.006628787878787879, + "matched_config_load_points": 33, + "max_absolute_error": 0.015625, + "signed_error_mean": -0.004734848484848485 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 6, + "feasibility_label_accuracy": 0.8181818181818182, + "mae": 0.01278409090909091, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.01278409090909091 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 891, + "mae_ms": 40.73985496630227, + "signed_error_ms_mean": -3.610411376010397 + }, + "[1024,2048)": { + "count": 693, + "mae_ms": 57.62684905459171, + "signed_error_ms_mean": 44.446875608364664 + }, + "[16384,32769)": { + "count": 132, + "mae_ms": 240.89426466203582, + "signed_error_ms_mean": 147.35987190047473 + }, + "[2048,4096)": { + "count": 132, + "mae_ms": 109.72593289069287, + "signed_error_ms_mean": 91.2075693704322 + }, + "[4096,8192)": { + "count": 99, + "mae_ms": 140.80238805214807, + "signed_error_ms_mean": 138.7995225585974 + }, + "[8192,16384)": { + "count": 165, + "mae_ms": 237.3733609970516, + "signed_error_ms_mean": 237.3733609970516 + } + }, + "mae_ms": 83.15460427483131, + "mape": 0.1633638442645967, + "matched_successful_requests": 2112, + "signed_error_ms_mean": 53.022475287000866 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 225, + "failure_set_jaccard": 0.7785467128027682, + "failure_set_union": 289, + "false_slo_fail_requests": 60, + "false_slo_pass_requests": 4, + "frontier_failure_requests": 285, + "label_accuracy": 0.9696969696969697, + "matched_requests": 2112, + "matching_labels": 2048, + "real_failure_requests": 229 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 129, + "failure_set_jaccard": 0.6172248803827751, + "failure_set_union": 209, + "false_slo_fail_requests": 62, + "false_slo_pass_requests": 18, + "frontier_failure_requests": 191, + "label_accuracy": 0.9621212121212122, + "matched_requests": 2112, + "matching_labels": 2032, + "real_failure_requests": 147 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 91, + "failure_set_jaccard": 0.8666666666666667, + "failure_set_union": 105, + "false_slo_fail_requests": 12, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 103, + "label_accuracy": 0.9933712121212122, + "matched_requests": 2112, + "matching_labels": 2098, + "real_failure_requests": 93 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 107, + "failure_set_jaccard": 0.7535211267605634, + "failure_set_union": 142, + "false_slo_fail_requests": 31, + "false_slo_pass_requests": 4, + "frontier_failure_requests": 138, + "label_accuracy": 0.9834280303030303, + "matched_requests": 2112, + "matching_labels": 2077, + "real_failure_requests": 111 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.890625, + 0.890625, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.890625, + 0.890625, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.875, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.90625, + 0.890625, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.90625, + 0.890625, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 8, + "real_tied_pairs": 12 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 8, + "real_tied_pairs": 12 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 2, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 4, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.9486832980505138, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.9375, + 0.921875, + 0.90625 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.9375, + 0.921875, + 0.90625 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 4, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9486832980505138, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 2, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + } + } + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json new file mode 100644 index 0000000..189ada2 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json @@ -0,0 +1,583 @@ +{ + "configs": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns128_mbt16384/engine.log", + "execution_index": 1, + "rate_order": [ + 0.75, + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns64_mbt16384/engine.log", + "execution_index": 2, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns64_mbt16384/engine.log", + "execution_index": 3, + "rate_order": [ + 0.35, + 0.25, + 0.3 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns64_mbt8192/engine.log", + "execution_index": 4, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns128_mbt8192/engine.log", + "execution_index": 5, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns128_mbt8192/engine.log", + "execution_index": 6, + "rate_order": [ + 0.15, + 0.35, + 0.25, + 0.3, + 0.2 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns64_mbt8192/engine.log", + "execution_index": 7, + "rate_order": [ + 0.25, + 0.35, + 0.15, + 0.3, + 0.2 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns128_mbt16384/engine.log", + "execution_index": 8, + "rate_order": [ + 0.35, + 0.25, + 0.3 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + } + ], + "created_unix_s": 1784142580.9845052, + "execution_contract": { + "config_order": "reverse_of_first_pass", + "estimated_replay_wall_seconds_excluding_model_loads": 6639.746031746032, + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "one_fresh_server_launch_per_config": true, + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", + "schema": "community-qwen235b-boundary-repeat-selection-v4", + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "selections": [ + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5, + 0.75 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.5, + 0.75 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns128_mbt16384", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.75, + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "source_relative_order": [ + 0.35, + 0.5, + 0.75 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns64_mbt16384", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.3 + ], + "boundary_rates": [ + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.25, + 0.35 + ] + }, + "config": "tp4_mns64_mbt16384", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.35, + 0.25, + 0.3 + ], + "selected_rates": [ + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns64_mbt8192", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns128_mbt8192", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.2, + 0.3 + ], + "boundary_rates": [ + 0.15, + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.15, + 0.25 + ] + }, + "config": "tp4_mns128_mbt8192", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.15, + 0.35, + 0.25, + 0.3, + 0.2 + ], + "selected_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35, + 0.15 + ] + }, + { + "added_refinement_rates": [ + 0.2, + 0.3 + ], + "boundary_rates": [ + 0.15, + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.15, + 0.25 + ] + }, + "config": "tp4_mns64_mbt8192", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.25, + 0.35, + 0.15, + 0.3, + 0.2 + ], + "selected_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.15, + 0.35, + 0.25 + ] + }, + { + "added_refinement_rates": [ + 0.3 + ], + "boundary_rates": [ + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.25, + 0.35 + ] + }, + "config": "tp4_mns128_mbt16384", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.35, + 0.25, + 0.3 + ], + "selected_rates": [ + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35 + ] + } + ], + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", + "warmup": { + "latencies_discarded": true, + "method": "one_disjoint_request_per_input_length_bin", + "request_count": 6, + "seed": 2026071504 + } + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "model": { + "config_sha256": "702c46d431bb984db9035a1225186bbfdb52c0d19c82104df4a37cd005e0369e", + "path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" + }, + "protocol_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/protocol_manifest.json", + "sha256": "695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb" + }, + "repeat_of": { + "first_pass_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "source_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/run_manifest.json", + "sha256": "6da81a34140d801cb6055e089bed7a12adb14c09dc87ca6dfa7f79df4ac34920" + } + }, + "repository": { + "commit": "501ceb817122747a23dc6d9fdc9d71dcb3aa8e72", + "status_porcelain": [] + }, + "runtime": { + "python": "/usr/bin/python3.12", + "vllm": "/tmp/wjh-frontier-vllm0102-smoke/.venv/bin/vllm" + }, + "schema": "community-qwen235b-prefill-fixed-cohort-v1", + "studies": { + "4": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json", + "sha256": "ea0d7d31e4a5102cf530e900754debb1d8b18f860748ead88ebf7de57d92faa3" + }, + "8": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json", + "sha256": "48c4d917b5531f742a05bab3bcd5c880f2eaddf9992b7629e9f48d8e00fd1906" + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json new file mode 100644 index 0000000..94ea5de --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json @@ -0,0 +1,732 @@ +{ + "created_unix_s": 1784153851.3812342, + "inputs": { + "first_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "repeat_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/community_ranking_frozen.json", + "sha256": "4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34" + } + }, + "primary_original_grid_capacity_stability": { + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "records": [ + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt16384", + "first_capacity": 0.3, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt8192", + "first_capacity": 0.2, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt16384", + "first_capacity": 0.3, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt8192", + "first_capacity": 0.2, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt8192", + "first_capacity": 0.45, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt8192", + "first_capacity": 0.45, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9933992677987828, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "stable_configs": 6, + "total_configs": 6 + }, + "real_trial_ttft_ms": { + "absolute_error_p95": 62.524845998268574, + "mae": 14.980625974885697, + "paired_successful_requests": 768, + "repeat_minus_first_mean": 1.5463623839574818 + }, + "schema": "community-qwen235b-trial-stability-v1", + "shared_config_load_points": 12, + "shared_requests": 768, + "variants": { + "legacy_step_1s_2s": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.45, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.45, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + } + ], + "frontier_correct_first": 742, + "frontier_correct_repeat": 742, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9661458333333334, + "frontier_request_label_accuracy_repeat": 0.9661458333333334, + "frontier_wrong_both_stable_real_label": 26, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_10k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 726, + "frontier_correct_repeat": 726, + "frontier_load_label_accuracy_first": 0.8333333333333334, + "frontier_load_label_accuracy_repeat": 0.8333333333333334, + "frontier_request_label_accuracy_first": 0.9453125, + "frontier_request_label_accuracy_repeat": 0.9453125, + "frontier_wrong_both_stable_real_label": 42, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_6k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 760, + "frontier_correct_repeat": 760, + "frontier_load_label_accuracy_first": 0.8333333333333334, + "frontier_load_label_accuracy_repeat": 0.8333333333333334, + "frontier_request_label_accuracy_first": 0.9895833333333334, + "frontier_request_label_accuracy_repeat": 0.9895833333333334, + "frontier_wrong_both_stable_real_label": 8, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_8k_primary": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 750, + "frontier_correct_repeat": 750, + "frontier_load_label_accuracy_first": 0.5, + "frontier_load_label_accuracy_repeat": 0.5, + "frontier_request_label_accuracy_first": 0.9765625, + "frontier_request_label_accuracy_repeat": 0.9765625, + "frontier_wrong_both_stable_real_label": 18, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json new file mode 100644 index 0000000..99aab5f --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json @@ -0,0 +1,345 @@ +{ + "configs": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns128_mbt16384/engine.log", + "execution_index": 1, + "rate_order": [ + 0.3, + 0.35 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns64_mbt8192/engine.log", + "execution_index": 2, + "rate_order": [ + 0.2, + 0.25 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns128_mbt8192/engine.log", + "execution_index": 3, + "rate_order": [ + 0.2, + 0.25 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp8_mns128_mbt8192/engine.log", + "execution_index": 4, + "rate_order": [ + 0.45, + 0.5 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp8_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp8_mns64_mbt8192/engine.log", + "execution_index": 5, + "rate_order": [ + 0.45, + 0.5 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp8_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns64_mbt16384/engine.log", + "execution_index": 6, + "rate_order": [ + 0.3, + 0.35 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + } + ], + "created_unix_s": 1784150504.0091515, + "execution_contract": { + "config_order": "reverse_of_first_pass", + "estimated_replay_wall_seconds_excluding_model_loads": 2484.8253968253966, + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "one_fresh_server_launch_per_config": true, + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", + "schema": "community-qwen235b-boundary-repeat-selection-v4", + "selected_configs": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "selected_slos": [ + "linear_8k_primary" + ], + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "selections": [ + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.3, + 0.35 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.3, + 0.35 + ] + }, + "config": "tp4_mns128_mbt16384", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.3, + 0.35 + ], + "selected_rates": [ + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.35, + 0.3 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.2, + 0.25 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.2, + 0.25 + ] + }, + "config": "tp4_mns64_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.2, + 0.25 + ], + "selected_rates": [ + 0.2, + 0.25 + ], + "source_relative_order": [ + 0.25, + 0.2 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.2, + 0.25 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.2, + 0.25 + ] + }, + "config": "tp4_mns128_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.2, + 0.25 + ], + "selected_rates": [ + 0.2, + 0.25 + ], + "source_relative_order": [ + 0.25, + 0.2 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.45, + 0.5 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.45, + 0.5 + ] + }, + "config": "tp8_mns128_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.45, + 0.5 + ], + "selected_rates": [ + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.5, + 0.45 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.45, + 0.5 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.45, + 0.5 + ] + }, + "config": "tp8_mns64_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.45, + 0.5 + ], + "selected_rates": [ + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.5, + 0.45 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.3, + 0.35 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.3, + 0.35 + ] + }, + "config": "tp4_mns64_mbt16384", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.3, + 0.35 + ], + "selected_rates": [ + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.35, + 0.3 + ] + } + ], + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", + "warmup": { + "latencies_discarded": true, + "method": "one_disjoint_request_per_input_length_bin", + "request_count": 6, + "seed": 2026071504 + } + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "model": { + "config_sha256": "702c46d431bb984db9035a1225186bbfdb52c0d19c82104df4a37cd005e0369e", + "path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" + }, + "protocol_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/protocol_manifest.json", + "sha256": "695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb" + }, + "repeat_of": { + "first_pass_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "source_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/run_manifest.json", + "sha256": "cf1a722043f42ff8f435e3521d0b2b2892a8ea0922e1e4870ec10fba26c0f998" + } + }, + "repository": { + "commit": "344af3a4285ccfdc6e10e540767808907ad81ddf", + "status_porcelain": [] + }, + "runtime": { + "python": "/usr/bin/python3.12", + "vllm": "/tmp/wjh-frontier-vllm0102-smoke/.venv/bin/vllm" + }, + "schema": "community-qwen235b-prefill-fixed-cohort-v1", + "studies": { + "4": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json", + "sha256": "ea0d7d31e4a5102cf530e900754debb1d8b18f860748ead88ebf7de57d92faa3" + }, + "8": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json", + "sha256": "48c4d917b5531f742a05bab3bcd5c880f2eaddf9992b7629e9f48d8e00fd1906" + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json new file mode 100644 index 0000000..485ca5e --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json @@ -0,0 +1,3830 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/community_ranking_frozen.json", + "sha256": "4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34" + }, + "created_unix_s": 1784153851.1556726, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.028645833333333332, + "matched_config_load_points": 12, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.028645833333333332 + }, + "linear_10k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.8333333333333334, + "mae": 0.028645833333333332, + "matched_config_load_points": 12, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.028645833333333332 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.8333333333333334, + "mae": 0.010416666666666666, + "matched_config_load_points": 12, + "max_absolute_error": 0.015625, + "signed_error_mean": -0.005208333333333333 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 6, + "feasibility_label_accuracy": 0.5, + "mae": 0.018229166666666668, + "matched_config_load_points": 12, + "max_absolute_error": 0.03125, + "signed_error_mean": -0.018229166666666668 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 324, + "mae_ms": 50.51891975082174, + "signed_error_ms_mean": -9.882795155125912 + }, + "[1024,2048)": { + "count": 252, + "mae_ms": 59.564509081632316, + "signed_error_ms_mean": 41.897250452193944 + }, + "[16384,32769)": { + "count": 48, + "mae_ms": 280.18396195154355, + "signed_error_ms_mean": 186.5034642541033 + }, + "[2048,4096)": { + "count": 48, + "mae_ms": 94.21679883383375, + "signed_error_ms_mean": 67.31957197829648 + }, + "[4096,8192)": { + "count": 36, + "mae_ms": 110.9597465520623, + "signed_error_ms_mean": 108.56515721844958 + }, + "[8192,16384)": { + "count": 60, + "mae_ms": 241.64142363249, + "signed_error_ms_mean": 241.64142363249 + } + }, + "mae_ms": 88.33679570229081, + "mape": 0.16159539870054299, + "matched_successful_requests": 768, + "signed_error_ms_mean": 49.40939882898548 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 82, + "failure_set_jaccard": 0.7592592592592593, + "failure_set_union": 108, + "false_slo_fail_requests": 24, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 106, + "label_accuracy": 0.9661458333333334, + "matched_requests": 768, + "matching_labels": 742, + "real_failure_requests": 84 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 52, + "failure_set_jaccard": 0.5531914893617021, + "failure_set_union": 94, + "false_slo_fail_requests": 32, + "false_slo_pass_requests": 10, + "frontier_failure_requests": 84, + "label_accuracy": 0.9453125, + "matched_requests": 768, + "matching_labels": 726, + "real_failure_requests": 62 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 34, + "failure_set_jaccard": 0.8095238095238095, + "failure_set_union": 42, + "false_slo_fail_requests": 6, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 40, + "label_accuracy": 0.9895833333333334, + "matched_requests": 768, + "matching_labels": 760, + "real_failure_requests": 36 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 42, + "failure_set_jaccard": 0.7, + "failure_set_union": 60, + "false_slo_fail_requests": 16, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 58, + "label_accuracy": 0.9765625, + "matched_requests": 768, + "matching_labels": 750, + "real_failure_requests": 44 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.917662935482247, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.875 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.875 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.917662935482247, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 8, + "correct_pairs": 8, + "frontier_tied_pairs": 7, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.8660254037844387, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.16666666666666663, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125 + ], + "rank": 3, + "slo": "linear_6k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": true + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125 + ], + "rank": 3, + "slo": "linear_6k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": true + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 8, + "correct_pairs": 8, + "frontier_tied_pairs": 7, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.8660254037844387, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.25 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.16666666666666663, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": false + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.9819805060619656, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + } + } + } + } +}