{
  "clean_window_failures": 0,
  "completed_burnins": 5,
  "completed_measured_runs": 40,
  "controller_pid": 2438791,
  "created_at": 1783833885.960389,
  "drain_quarantined_runs": 0,
  "fingerprint": {
    "cells": [
      "P08-C00",
      "P03-C10",
      "P07-C00",
      "P10-C01",
      "P01-C10",
      "P01-C01",
      "P10-C10",
      "P03-C01",
      "P09-C00",
      "P06-C01",
      "P10-C11",
      "P01-C00",
      "P01-C11",
      "P10-C00",
      "P04-C00",
      "P03-C00",
      "P06-C10",
      "P02-C00",
      "P06-C00",
      "P06-C11",
      "P11-C00",
      "P10-C00-TP2",
      "P03-C11",
      "P05-C00"
    ],
    "client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
    "common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
    "controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
    "cpu_map": {
      "0": "0-19",
      "1": "20-39",
      "2": "40-59",
      "3": "60-79",
      "4": "80-99",
      "5": "100-119",
      "6": "120-139",
      "7": "140-159"
    },
    "manifests": {
      "P01": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
        "rows": 32768,
        "sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
      },
      "P02": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
        "rows": 32768,
        "sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
      },
      "P03": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
        "rows": 32768,
        "sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
      },
      "P04": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
        "rows": 32768,
        "sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
      },
      "P05": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
        "rows": 32768,
        "sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
      },
      "P06": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
        "rows": 32768,
        "sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
      },
      "P07": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
        "rows": 32768,
        "sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
      },
      "P08": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
        "rows": 32768,
        "sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
      },
      "P09": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
        "rows": 32768,
        "sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
      },
      "P10": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
        "rows": 4011,
        "sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
      },
      "P11": {
        "path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
        "rows": 32768,
        "sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
      }
    },
    "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
    "source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
    "source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
  },
  "gpu_hours_this_stage": 0.7173924536837472,
  "gpu_hours_total": 14.025875418755744,
  "missing_trace_files": 8,
  "repairs": [
    {
      "change": "dedicated aiohttp TCPConnector(limit=2) for profile endpoint session; request stream unchanged",
      "evidence": "/start_profile connection acquisition timed out before server receipt while 256 data-plane connections were occupied",
      "failed_run": "P03-C01-saturation",
      "failed_stage": "primary-02-saturation",
      "new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
      "old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
      "repair": "profile-control-connector-isolation",
      "retry": "one exact whole-wave retry; failed directories retained as .interrupted-*",
      "schema": 1,
      "tests": {
        "py_compile": "PASS",
        "ruff": "PASS",
        "unittest": "13/13 PASS"
      },
      "timestamp": 1783837763.431071
    },
    {
      "budget": {
        "hard_cap": 16.0,
        "projected_remaining_h20_hours": 6.3857560787267165,
        "projected_total_h20_hours": 14.513826778670154,
        "projected_total_with_15pct_remaining_contingency": 15.471690190479162,
        "used_h20_hours": 8.128070699943438
      },
      "cause": "controller required failed_records==0 across excluded profile/recovery time although the registered hard gate is zero clean-window failures",
      "change": "replace all_failures_zero with clean-window boundary validation plus failed-record accounting; excluded failures remain reported",
      "evidence": {
        "clean_completed": 5828,
        "clean_failed": 0,
        "error_kind": "ServerDisconnectedError",
        "excluded_window_failures": 2,
        "failure_completion_s": [
          373.1505718010012,
          373.9740898209857
        ],
        "run": "P01-C01-moderate"
      },
      "failed_stage": "primary-02-moderate",
      "gpu_rerun": false,
      "new_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
      "old_controller_sha256": "167e48f98f307e16ee44b321068a82a13813b3bf9a4d76882f774f15fe85e595",
      "repair": "clean-window-failure-scope",
      "schema": 1,
      "tests": {
        "py_compile": "PASS",
        "ruff": "PASS",
        "unittest": "14/14 PASS"
      },
      "timestamp": 1783847092.6920242
    },
    {
      "cause": "single preflight NVML sample observed 4 MiB on GPU0-3 with 0% utilization and no compute apps immediately after a verified cleanup",
      "change": "bounded 60s preflight requiring three consecutive samples with zero memory, zero utilization, and no compute apps; no nonzero-memory tolerance",
      "failed_stage": "primary-06-saturation",
      "gpu_hours_total_unchanged": 11.920448313262728,
      "gpu_launch_before_failure": false,
      "new_common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
      "old_common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
      "repair": "stable-zero-preflight",
      "schema": 1,
      "tests": {
        "py_compile": "PASS",
        "ruff": "PASS",
        "unittest": "15/15 PASS"
      },
      "timestamp": 1783850898.1496143
    },
    {
      "code_change": false,
      "gpu_hours_used": 12.606695837948058,
      "hard_cap": 16.0,
      "projected_remaining_h20_hours": 2.1455133807990285,
      "projected_total_with_15pct_contingency": 15.07403622586694,
      "protocol_action": "single exact whole-wave infrastructure retry under identical 4-way placement; gate unchanged",
      "reason": "first TP2 P10 inference-shape cold start/autotune after readiness yielded 15 successful warmup completions against required 32; clean window and artifacts otherwise valid",
      "repair": "tp2-warmup-exact-retry",
      "run": "P10-C00-TP2-saturation",
      "schema": 1,
      "stage": "primary-06-saturation",
      "timestamp": 1783851710.3100822,
      "warmup_required": 32,
      "warmup_success": 15
    },
    {
      "accepted_runs_preserved": 40,
      "amendment": "A-P3-6",
      "changed_fingerprint_keys": [
        "controller_sha256"
      ],
      "created_at": 1783853258.3098204,
      "failed_stage_preserved": "primary-06-saturation",
      "new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
      "old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
      "projected_remaining_h20_hours": 2.1,
      "projected_total_h20_hours": 15.408482965071997,
      "repair_id": "repair-008-ap36",
      "retained_attempt_re_adjudicated": false,
      "retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
      "schema": 1
    }
  ],
  "schema": 1,
  "stages": {
    "burnin-01": {
      "assignments": [
        {
          "cell": "P06-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P06-C10",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P06-C01",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P06-C11",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": true,
      "clients": {},
      "completed_at": 1783834288.1238313,
      "confirmation": false,
      "gpu_hours": 0.30808498481909435,
      "load_point": "saturation",
      "profile": false,
      "servers": {},
      "started_at": 1783833886.4448946,
      "status": "complete",
      "validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
    },
    "burnin-02": {
      "assignments": [
        {
          "cell": "P06-C00-TP2",
          "gpus": [
            0,
            1
          ]
        }
      ],
      "burnin": true,
      "clients": {},
      "completed_at": 1783834671.921921,
      "confirmation": false,
      "gpu_hours": 0.19903395679261948,
      "load_point": "saturation",
      "profile": false,
      "servers": {},
      "started_at": 1783834301.7813473,
      "status": "complete"
    },
    "primary-01-moderate": {
      "assignments": [
        {
          "cell": "P08-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P03-C10",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P07-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P10-C01",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783836723.3639715,
      "confirmation": false,
      "gpu_hours": 0.5195140059126748,
      "load_point": "moderate",
      "profile": true,
      "servers": {},
      "started_at": 1783836150.2169476,
      "status": "complete",
      "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
      "warmup_feasibility": "P10 target=min(32,floor(0.445*60))=26; observed=27"
    },
    "primary-01-saturation": {
      "assignments": [
        {
          "cell": "P08-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P03-C10",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P07-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P10-C01",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783836130.8498049,
      "confirmation": false,
      "drain_readjudication": "P10/C01 288.619107924s <= 600s",
      "gpu_hours": 0.8836704309119119,
      "load_point": "saturation",
      "missing_trace_files": 8,
      "profile": true,
      "servers": {},
      "started_at": 1783834671.9563339,
      "status": "complete",
      "validation_attempt1_failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")"
    },
    "primary-02-moderate": {
      "assignments": [
        {
          "cell": "P01-C10",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P01-C01",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P10-C10",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P03-C01",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783847092.7229948,
      "confirmation": false,
      "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
      "gpu_hours": 0.5151156750652525,
      "load_point": "moderate",
      "profile": true,
      "readjudicated_without_gpu": true,
      "servers": {},
      "started_at": 1783838513.4472992,
      "status": "complete",
      "validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")"
    },
    "primary-02-saturation": {
      "assignments": [
        {
          "cell": "P01-C10",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P01-C01",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P10-C10",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P03-C01",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783838513.4027848,
      "confirmation": false,
      "gpu_hours": 0.7973502041896184,
      "load_point": "saturation",
      "profile": true,
      "servers": {},
      "started_at": 1783837773.6088765,
      "status": "complete"
    },
    "primary-03-moderate": {
      "assignments": [
        {
          "cell": "P09-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P06-C01",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P10-C11",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P01-C00",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783848369.3507695,
      "confirmation": false,
      "gpu_hours": 0.5205460974905226,
      "load_point": "moderate",
      "profile": true,
      "servers": {},
      "started_at": 1783847881.06146,
      "status": "complete"
    },
    "primary-03-saturation": {
      "assignments": [
        {
          "cell": "P09-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P06-C01",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P10-C11",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P01-C00",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783847881.0197804,
      "confirmation": false,
      "gpu_hours": 0.8273645816908942,
      "load_point": "saturation",
      "profile": true,
      "servers": {},
      "started_at": 1783847109.5683346,
      "status": "complete"
    },
    "primary-04-moderate": {
      "assignments": [
        {
          "cell": "P01-C11",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P10-C00",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P04-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P03-C00",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783849583.5228572,
      "confirmation": false,
      "gpu_hours": 0.5205216948853598,
      "load_point": "moderate",
      "profile": true,
      "servers": {},
      "started_at": 1783849096.4363403,
      "status": "complete"
    },
    "primary-04-saturation": {
      "assignments": [
        {
          "cell": "P01-C11",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P10-C00",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P04-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P03-C00",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783849096.3948114,
      "confirmation": false,
      "gpu_hours": 0.7872003297011058,
      "load_point": "saturation",
      "profile": true,
      "servers": {},
      "started_at": 1783848369.390857,
      "status": "complete"
    },
    "primary-05-moderate": {
      "assignments": [
        {
          "cell": "P06-C10",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P02-C00",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P06-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P06-C11",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783850641.6781306,
      "confirmation": false,
      "gpu_hours": 0.5138198028008143,
      "load_point": "moderate",
      "profile": true,
      "servers": {},
      "started_at": 1783850163.1211865,
      "status": "complete"
    },
    "primary-05-saturation": {
      "assignments": [
        {
          "cell": "P06-C10",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P02-C00",
          "gpus": [
            1
          ]
        },
        {
          "cell": "P06-C00",
          "gpus": [
            2
          ]
        },
        {
          "cell": "P06-C11",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {},
      "completed_at": 1783850163.0472832,
      "confirmation": false,
      "gpu_hours": 0.6229251067505942,
      "load_point": "saturation",
      "profile": true,
      "servers": {},
      "started_at": 1783849583.5652869,
      "status": "complete"
    },
    "primary-06-saturation": {
      "assignments": [
        {
          "cell": "P11-C00",
          "gpus": [
            0
          ]
        },
        {
          "cell": "P10-C00-TP2",
          "gpus": [
            1,
            2
          ]
        },
        {
          "cell": "P03-C11",
          "gpus": [
            3
          ]
        }
      ],
      "burnin": false,
      "clients": {
        "P03-C11-saturation": {
          "pgid": 2440965,
          "pid": 2440965
        },
        "P10-C00-TP2-saturation": {
          "pgid": 2440964,
          "pid": 2440964
        },
        "P11-C00-saturation": {
          "pgid": 2440962,
          "pid": 2440962
        }
      },
      "confirmation": false,
      "failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C00-TP2/saturation: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=17; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [11, 10, 16], 'scheduled_tokens': [90112, 81920, 70313], 'scheduled_token_throughput': [18022.4, 16384.0, 14062.6], 'mean_scheduled_token_throughput': 16156.333333333334, 'slope_tokens_per_second_squared': -395.98000000000013, 'normalized_drift': 0.367639109533929, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
      "gpu_hours": 0.7173924536837472,
      "load_point": "saturation",
      "profile": true,
      "servers": {
        "P03-C11-saturation": {
          "gpus": [
            3
          ],
          "pgid": 2438868,
          "pid": 2438868
        },
        "P10-C00-TP2-saturation": {
          "gpus": [
            1,
            2
          ],
          "pgid": 2438867,
          "pid": 2438867
        },
        "P11-C00-saturation": {
          "gpus": [
            0
          ],
          "pgid": 2438866,
          "pid": 2438866
        }
      },
      "started_at": 1783853279.9695158,
      "status": "failed"
    }
  },
  "status": "failed",
  "updated_at": 1783853946.066117
}
