{ "clean_window_failures": 0, "completed_burnins": 5, "completed_measured_runs": 40, "controller_pid": 2438791, "created_at": 1783833885.960389, "drain_quarantined_runs": 0, "fingerprint": { "cells": [ "P08-C00", "P03-C10", "P07-C00", "P10-C01", "P01-C10", "P01-C01", "P10-C10", "P03-C01", "P09-C00", "P06-C01", "P10-C11", "P01-C00", "P01-C11", "P10-C00", "P04-C00", "P03-C00", "P06-C10", "P02-C00", "P06-C00", "P06-C11", "P11-C00", "P10-C00-TP2", "P03-C11", "P05-C00" ], "client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0", "common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f", "controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f", "cpu_map": { "0": "0-19", "1": "20-39", "2": "40-59", "3": "60-79", "4": "80-99", "5": "100-119", "6": "120-139", "7": "140-159" }, "manifests": { "P01": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl", "rows": 32768, "sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4" }, "P02": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl", "rows": 32768, "sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9" }, "P03": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl", "rows": 32768, "sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1" }, "P04": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl", "rows": 32768, "sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247" }, "P05": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl", "rows": 32768, "sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7" }, "P06": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl", "rows": 32768, "sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96" }, "P07": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl", "rows": 32768, "sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6" }, "P08": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl", "rows": 32768, "sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9" }, "P09": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl", "rows": 32768, "sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e" }, "P10": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl", "rows": 4011, "sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33" }, "P11": { "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl", "rows": 32768, "sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18" } }, "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", "source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05", "source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1" }, "gpu_hours_this_stage": 0.7173924536837472, "gpu_hours_total": 14.025875418755744, "missing_trace_files": 8, "repairs": [ { "change": "dedicated aiohttp TCPConnector(limit=2) for profile endpoint session; request stream unchanged", "evidence": "/start_profile connection acquisition timed out before server receipt while 256 data-plane connections were occupied", "failed_run": "P03-C01-saturation", "failed_stage": "primary-02-saturation", "new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0", "old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84", "repair": "profile-control-connector-isolation", "retry": "one exact whole-wave retry; failed directories retained as .interrupted-*", "schema": 1, "tests": { "py_compile": "PASS", "ruff": "PASS", "unittest": "13/13 PASS" }, "timestamp": 1783837763.431071 }, { "budget": { "hard_cap": 16.0, "projected_remaining_h20_hours": 6.3857560787267165, "projected_total_h20_hours": 14.513826778670154, "projected_total_with_15pct_remaining_contingency": 15.471690190479162, "used_h20_hours": 8.128070699943438 }, "cause": "controller required failed_records==0 across excluded profile/recovery time although the registered hard gate is zero clean-window failures", "change": "replace all_failures_zero with clean-window boundary validation plus failed-record accounting; excluded failures remain reported", "evidence": { "clean_completed": 5828, "clean_failed": 0, "error_kind": "ServerDisconnectedError", "excluded_window_failures": 2, "failure_completion_s": [ 373.1505718010012, 373.9740898209857 ], "run": "P01-C01-moderate" }, "failed_stage": "primary-02-moderate", "gpu_rerun": false, "new_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f", "old_controller_sha256": "167e48f98f307e16ee44b321068a82a13813b3bf9a4d76882f774f15fe85e595", "repair": "clean-window-failure-scope", "schema": 1, "tests": { "py_compile": "PASS", "ruff": "PASS", "unittest": "14/14 PASS" }, "timestamp": 1783847092.6920242 }, { "cause": "single preflight NVML sample observed 4 MiB on GPU0-3 with 0% utilization and no compute apps immediately after a verified cleanup", "change": "bounded 60s preflight requiring three consecutive samples with zero memory, zero utilization, and no compute apps; no nonzero-memory tolerance", "failed_stage": "primary-06-saturation", "gpu_hours_total_unchanged": 11.920448313262728, "gpu_launch_before_failure": false, "new_common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f", "old_common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5", "repair": "stable-zero-preflight", "schema": 1, "tests": { "py_compile": "PASS", "ruff": "PASS", "unittest": "15/15 PASS" }, "timestamp": 1783850898.1496143 }, { "code_change": false, "gpu_hours_used": 12.606695837948058, "hard_cap": 16.0, "projected_remaining_h20_hours": 2.1455133807990285, "projected_total_with_15pct_contingency": 15.07403622586694, "protocol_action": "single exact whole-wave infrastructure retry under identical 4-way placement; gate unchanged", "reason": "first TP2 P10 inference-shape cold start/autotune after readiness yielded 15 successful warmup completions against required 32; clean window and artifacts otherwise valid", "repair": "tp2-warmup-exact-retry", "run": "P10-C00-TP2-saturation", "schema": 1, "stage": "primary-06-saturation", "timestamp": 1783851710.3100822, "warmup_required": 32, "warmup_success": 15 }, { "accepted_runs_preserved": 40, "amendment": "A-P3-6", "changed_fingerprint_keys": [ "controller_sha256" ], "created_at": 1783853258.3098204, "failed_stage_preserved": "primary-06-saturation", "new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f", "old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f", "projected_remaining_h20_hours": 2.1, "projected_total_h20_hours": 15.408482965071997, "repair_id": "repair-008-ap36", "retained_attempt_re_adjudicated": false, "retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44", "schema": 1 } ], "schema": 1, "stages": { "burnin-01": { "assignments": [ { "cell": "P06-C00", "gpus": [ 0 ] }, { "cell": "P06-C10", "gpus": [ 1 ] }, { "cell": "P06-C01", "gpus": [ 2 ] }, { "cell": "P06-C11", "gpus": [ 3 ] } ], "burnin": true, "clients": {}, "completed_at": 1783834288.1238313, "confirmation": false, "gpu_hours": 0.30808498481909435, "load_point": "saturation", "profile": false, "servers": {}, "started_at": 1783833886.4448946, "status": "complete", "validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")" }, "burnin-02": { "assignments": [ { "cell": "P06-C00-TP2", "gpus": [ 0, 1 ] } ], "burnin": true, "clients": {}, "completed_at": 1783834671.921921, "confirmation": false, "gpu_hours": 0.19903395679261948, "load_point": "saturation", "profile": false, "servers": {}, "started_at": 1783834301.7813473, "status": "complete" }, "primary-01-moderate": { "assignments": [ { "cell": "P08-C00", "gpus": [ 0 ] }, { "cell": "P03-C10", "gpus": [ 1 ] }, { "cell": "P07-C00", "gpus": [ 2 ] }, { "cell": "P10-C01", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783836723.3639715, "confirmation": false, "gpu_hours": 0.5195140059126748, "load_point": "moderate", "profile": true, "servers": {}, "started_at": 1783836150.2169476, "status": "complete", "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")", "warmup_feasibility": "P10 target=min(32,floor(0.445*60))=26; observed=27" }, "primary-01-saturation": { "assignments": [ { "cell": "P08-C00", "gpus": [ 0 ] }, { "cell": "P03-C10", "gpus": [ 1 ] }, { "cell": "P07-C00", "gpus": [ 2 ] }, { "cell": "P10-C01", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783836130.8498049, "confirmation": false, "drain_readjudication": "P10/C01 288.619107924s <= 600s", "gpu_hours": 0.8836704309119119, "load_point": "saturation", "missing_trace_files": 8, "profile": true, "servers": {}, "started_at": 1783834671.9563339, "status": "complete", "validation_attempt1_failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")" }, "primary-02-moderate": { "assignments": [ { "cell": "P01-C10", "gpus": [ 0 ] }, { "cell": "P01-C01", "gpus": [ 1 ] }, { "cell": "P10-C10", "gpus": [ 2 ] }, { "cell": "P03-C01", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783847092.7229948, "confirmation": false, "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")", "gpu_hours": 0.5151156750652525, "load_point": "moderate", "profile": true, "readjudicated_without_gpu": true, "servers": {}, "started_at": 1783838513.4472992, "status": "complete", "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")" }, "primary-02-saturation": { "assignments": [ { "cell": "P01-C10", "gpus": [ 0 ] }, { "cell": "P01-C01", "gpus": [ 1 ] }, { "cell": "P10-C10", "gpus": [ 2 ] }, { "cell": "P03-C01", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783838513.4027848, "confirmation": false, "gpu_hours": 0.7973502041896184, "load_point": "saturation", "profile": true, "servers": {}, "started_at": 1783837773.6088765, "status": "complete" }, "primary-03-moderate": { "assignments": [ { "cell": "P09-C00", "gpus": [ 0 ] }, { "cell": "P06-C01", "gpus": [ 1 ] }, { "cell": "P10-C11", "gpus": [ 2 ] }, { "cell": "P01-C00", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783848369.3507695, "confirmation": false, "gpu_hours": 0.5205460974905226, "load_point": "moderate", "profile": true, "servers": {}, "started_at": 1783847881.06146, "status": "complete" }, "primary-03-saturation": { "assignments": [ { "cell": "P09-C00", "gpus": [ 0 ] }, { "cell": "P06-C01", "gpus": [ 1 ] }, { "cell": "P10-C11", "gpus": [ 2 ] }, { "cell": "P01-C00", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783847881.0197804, "confirmation": false, "gpu_hours": 0.8273645816908942, "load_point": "saturation", "profile": true, "servers": {}, "started_at": 1783847109.5683346, "status": "complete" }, "primary-04-moderate": { "assignments": [ { "cell": "P01-C11", "gpus": [ 0 ] }, { "cell": "P10-C00", "gpus": [ 1 ] }, { "cell": "P04-C00", "gpus": [ 2 ] }, { "cell": "P03-C00", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783849583.5228572, "confirmation": false, "gpu_hours": 0.5205216948853598, "load_point": "moderate", "profile": true, "servers": {}, "started_at": 1783849096.4363403, "status": "complete" }, "primary-04-saturation": { "assignments": [ { "cell": "P01-C11", "gpus": [ 0 ] }, { "cell": "P10-C00", "gpus": [ 1 ] }, { "cell": "P04-C00", "gpus": [ 2 ] }, { "cell": "P03-C00", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783849096.3948114, "confirmation": false, "gpu_hours": 0.7872003297011058, "load_point": "saturation", "profile": true, "servers": {}, "started_at": 1783848369.390857, "status": "complete" }, "primary-05-moderate": { "assignments": [ { "cell": "P06-C10", "gpus": [ 0 ] }, { "cell": "P02-C00", "gpus": [ 1 ] }, { "cell": "P06-C00", "gpus": [ 2 ] }, { "cell": "P06-C11", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783850641.6781306, "confirmation": false, "gpu_hours": 0.5138198028008143, "load_point": "moderate", "profile": true, "servers": {}, "started_at": 1783850163.1211865, "status": "complete" }, "primary-05-saturation": { "assignments": [ { "cell": "P06-C10", "gpus": [ 0 ] }, { "cell": "P02-C00", "gpus": [ 1 ] }, { "cell": "P06-C00", "gpus": [ 2 ] }, { "cell": "P06-C11", "gpus": [ 3 ] } ], "burnin": false, "clients": {}, "completed_at": 1783850163.0472832, "confirmation": false, "gpu_hours": 0.6229251067505942, "load_point": "saturation", "profile": true, "servers": {}, "started_at": 1783849583.5652869, "status": "complete" }, "primary-06-saturation": { "assignments": [ { "cell": "P11-C00", "gpus": [ 0 ] }, { "cell": "P10-C00-TP2", "gpus": [ 1, 2 ] }, { "cell": "P03-C11", "gpus": [ 3 ] } ], "burnin": false, "clients": { "P03-C11-saturation": { "pgid": 2440965, "pid": 2440965 }, "P10-C00-TP2-saturation": { "pgid": 2440964, "pid": 2440964 }, "P11-C00-saturation": { "pgid": 2440962, "pid": 2440962 } }, "confirmation": false, "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C00-TP2/saturation: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=17; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [11, 10, 16], 'scheduled_tokens': [90112, 81920, 70313], 'scheduled_token_throughput': [18022.4, 16384.0, 14062.6], 'mean_scheduled_token_throughput': 16156.333333333334, 'slope_tokens_per_second_squared': -395.98000000000013, 'normalized_drift': 0.367639109533929, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")", "gpu_hours": 0.7173924536837472, "load_point": "saturation", "profile": true, "servers": { "P03-C11-saturation": { "gpus": [ 3 ], "pgid": 2438868, "pid": 2438868 }, "P10-C00-TP2-saturation": { "gpus": [ 1, 2 ], "pgid": 2438867, "pid": 2438867 }, "P11-C00-saturation": { "gpus": [ 0 ], "pgid": 2438866, "pid": 2438866 } }, "started_at": 1783853279.9695158, "status": "failed" } }, "status": "failed", "updated_at": 1783853946.066117 }