62 lines
4.3 KiB
JSON
62 lines
4.3 KiB
JSON
{
|
|
"schema": "qwen235b-t0-real-exclusions-v1",
|
|
"policy": "Exclude the complete output directory whenever another process shares any allocated GPU or benchmark endpoint during warmup or a measured anchor.",
|
|
"excluded_attempts": [
|
|
{
|
|
"reason": "Fleet monitor re-probed during model load and oversubscribed both TP4 allocations; the later MNS128 attempts overlapped the MNS64 warmup and start of r0p10.",
|
|
"remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/contaminated-queue-race-20260716T045500Z",
|
|
"run_ids": [
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T045338281933Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T045339503446Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1-20260716T045458846499Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1-20260716T045500253803Z"
|
|
]
|
|
},
|
|
{
|
|
"reason": "A dispatch command still copying the source was mistakenly resubmitted. The duplicate controllers reused the first pair's open ports and issued concurrent warmup requests, so both the original and duplicate outputs were rejected.",
|
|
"remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/contaminated-duplicate-dispatch-20260716T0501Z",
|
|
"run_ids": [
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050017477963Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050018748070Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050123485776Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050124800483Z"
|
|
]
|
|
},
|
|
{
|
|
"reason": "Clean diagnostic run exposed cross-anchor execution-state leakage: TPOT at 1.60 req/s failed after a sustained 0.10 anchor but passed after a 2.40 anchor. The multi-rate-per-server contract does not match Frontier's independent rate runs, so the complete attempt is diagnostic-only.",
|
|
"remote_quarantine": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/artifacts/diagnostic-cross-anchor-state-20260716T0504Z",
|
|
"run_ids": [
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T050405703312Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T050407104293Z"
|
|
]
|
|
}
|
|
],
|
|
"accepted_attempts": [
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-20260716-v1-20260716T054640909739Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-20260716-v1-20260716T054642115787Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt8192-expansion-r3p20-20260716-v1-20260716T062904775741Z",
|
|
"qwen235b-t0-real-tp4_mns64_mbt16384-expansion-r3p20-20260716-v1-20260716T062906244819Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt8192-20260716-v1-20260716T063826749477Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt16384-20260716-v1-20260716T063827805737Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt8192-expansion-r3p20-20260716-v1-20260716T072121119478Z",
|
|
"qwen235b-t0-real-tp4_mns128_mbt16384-expansion-r3p20-20260716-v1-20260716T072122501851Z",
|
|
"qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1-20260716T073151443129Z",
|
|
"qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r2p40-20260716-v1-20260716T081536141326Z",
|
|
"qwen235b-t0-real-tp8_mns64_mbt8192-expansion-r3p20-20260716-v1-20260716T082603837807Z",
|
|
"qwen235b-t0-real-tp8_mns64_mbt16384-20260716-v1-20260716T083632620015Z",
|
|
"qwen235b-t0-real-tp8_mns64_mbt16384-expansion-r2p40-20260716-v1-20260716T092040851202Z",
|
|
"qwen235b-t0-real-tp8_mns128_mbt8192-20260716-v1-20260716T093245831612Z",
|
|
"qwen235b-t0-real-tp8_mns128_mbt8192-expansion-r2p40-20260716-v1-20260716T101734683241Z",
|
|
"qwen235b-t0-real-tp8_mns128_mbt16384-20260716-v1-20260716T102855604708Z",
|
|
"qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r2p40-20260716-v1-20260716T111437770615Z",
|
|
"qwen235b-t0-real-tp8_mns128_mbt16384-expansion-r3p20-20260716-v1-20260716T112608807037Z"
|
|
],
|
|
"accepted_warnings": [
|
|
{
|
|
"run_id": "qwen235b-t0-real-tp8_mns64_mbt8192-20260716-v1-20260716T073151443129Z",
|
|
"anchor": "round1/r1p20",
|
|
"reason": "One rank-7 TCPStore heartbeat warning occurred 1.1 s after the complete 64-request result was written, while all eight workers were terminating after the runner intentionally stopped the fresh server. The application then shut down normally; no request, engine, OOM, or in-window NCCL failure occurred."
|
|
}
|
|
]
|
|
}
|