Traceback (most recent call last):
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 445, in <module>
    main()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 434, in main
    execute_wave(index, state, expected)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 410, in execute_wave
    raise failure
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 329, in execute_wave
    wait_ready(entry)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 142, in wait_ready
    assert_no_other_compute()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 111, in assert_no_other_compute
    raise RuntimeError(f"outside GPU processes detected: {other}")
RuntimeError: outside GPU processes detected: [2635013, 2635031, 2635032, 2635034]
Traceback (most recent call last):
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 452, in <module>
    main()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 441, in main
    execute_wave(index, state, expected)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 418, in execute_wave
    validations = [validate_cell(entry) for entry in entries]
                   ^^^^^^^^^^^^^^^^^^^^
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 280, in validate_cell
    raise RuntimeError(f"cell validity failure {entry['cell']}: {invariants}")
RuntimeError: cell validity failure tp1_mns8: {'one_ready_marker': True, 'compile_capture_pre_ready': True, 'warmup_exact_16': True, 'warmup_long': True, 'layer1_footer': False, 'layer1_contiguous': True, 'layer1_zero_drops': False, 'sidecar_final': False, 'sidecar_agrees': False, 'anchor_intervals_present': True}
WAVE_ECHO wave=W2-tp2 assignments=tp2_mns8:gpu0+1,tp2_mns16:gpu2+3,tp2_mns32:gpu4+5,tp2_mns64:gpu6+7 spent_h20h=0.432311 wave_est_h20h=0.650 remaining_projection_h20h=2.250 cap_h20h=3.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json workload=chat_w20260311_1000.jsonl
WAVE_ECHO wave=W3-tp4-trap assignments=tp4_mns8:gpu0+1+2+3,tp4_mns16:gpu4+5+6+7 spent_h20h=1.352676 wave_est_h20h=0.850 remaining_projection_h20h=1.600 cap_h20h=3.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json workload=chat_w20260311_1000.jsonl
Traceback (most recent call last):
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 492, in <module>
    main()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 481, in main
    execute_wave(index, state, expected)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 334, in execute_wave
    raise RuntimeError(f"projected budget exceeds cap before {wave_name}: {state['gpu_hours_total']+future}")
RuntimeError: projected budget exceeds cap before W4-tp4: 3.0411728494699863