SOLO_WAVE_ECHO cell=tp4_mns32 tp=4 mns=32 gpus=0-3 mandatory=0.033717411016 spent_h20h=2.291173 cell_est_h20h=0.480 remaining_projection_h20h=3.440 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp4_mns64 tp=4 mns=64 gpus=0-3 mandatory=0.033717411016 spent_h20h=2.805712 cell_est_h20h=0.480 remaining_projection_h20h=2.960 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
Traceback (most recent call last):
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 406, in <module>
    main()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 392, in main
    execute_cell(index, cell, state, expected, histories)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 374, in execute_cell
    raise failure
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 361, in execute_cell
    base.assert_all_idle()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 133, in assert_all_idle
    raise RuntimeError(f"GPU cleanup failure: {bad}")
RuntimeError: GPU cleanup failure: [(0, 4, 0), (1, 4, 0), (2, 4, 0), (3, 4, 0), (4, 4, 0), (5, 4, 0), (6, 4, 0), (7, 1, 0)]
SOLO_WAVE_ECHO cell=tp2_mns32 tp=2 mns=32 gpus=0-1 mandatory=0.75390625,0.75 spent_h20h=3.315064 cell_est_h20h=0.220 remaining_projection_h20h=2.480 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp2_mns64 tp=2 mns=64 gpus=0-1 mandatory=0.75,0.5 spent_h20h=3.490967 cell_est_h20h=0.220 remaining_projection_h20h=2.260 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp4_mns16 tp=4 mns=16 gpus=0-3 mandatory=0.033717411016,0.033182214016,0.034252608017 spent_h20h=3.664183 cell_est_h20h=0.480 remaining_projection_h20h=2.040 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp2_mns8 tp=2 mns=8 gpus=0-1 mandatory=0.49609375,0.4921875 spent_h20h=4.217542 cell_est_h20h=0.220 remaining_projection_h20h=1.560 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp2_mns16 tp=2 mns=16 gpus=0-1 mandatory=0.49609375,0.4921875 spent_h20h=4.392992 cell_est_h20h=0.220 remaining_projection_h20h=1.340 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp4_mns8 tp=4 mns=8 gpus=0-3 mandatory=0.016591107009,0.016055910008 spent_h20h=4.623573 cell_est_h20h=0.480 remaining_projection_h20h=1.120 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp1_mns8 tp=1 mns=8 gpus=0-0 mandatory=0.2265625,0.21875 spent_h20h=5.120182 cell_est_h20h=0.110 remaining_projection_h20h=0.640 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp1_mns16 tp=1 mns=16 gpus=0-0 mandatory=0.24609375,0.25 spent_h20h=5.319979 cell_est_h20h=0.110 remaining_projection_h20h=0.530 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp1_mns32 tp=1 mns=32 gpus=0-0 mandatory=0.2421875,0.24609375 spent_h20h=5.404469 cell_est_h20h=0.110 remaining_projection_h20h=0.420 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
SOLO_WAVE_ECHO cell=tp1_mns64 tp=1 mns=64 gpus=0-0 mandatory=0.2421875,0.24609375 spent_h20h=5.523280 cell_est_h20h=0.110 remaining_projection_h20h=0.310 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
Traceback (most recent call last):
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 419, in <module>
    main()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 405, in main
    execute_cell(index, cell, state, expected, histories)
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_solo_controller.py", line 295, in execute_cell
    base.assert_all_idle()
  File "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_controller.py", line 133, in assert_all_idle
    raise RuntimeError(f"GPU cleanup failure: {bad}")
RuntimeError: GPU cleanup failure: [(0, 4, 0), (1, 4, 0), (2, 1, 0), (3, 1, 0), (4, 1, 0), (5, 1, 0), (6, 1, 0), (7, 1, 0)]
SOLO_WAVE_ECHO cell=tp1_mns64 tp=1 mns=64 gpus=0-0 mandatory=0.2421875,0.24609375 spent_h20h=5.523280 cell_est_h20h=0.110 remaining_projection_h20h=0.310 cap_h20h=6.0 ground_truth=/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/ground_truth.json trace=/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl
{"campaign_gpu_hours": 5.644544164273745, "cells": 12, "confirmations": 29, "primary_anchors": 37, "solo_gpu_hours": 3.3533713148037596, "status": "complete"}
