diff --git a/scripts/sweep_e4_pressured.sh b/scripts/sweep_e4_pressured.sh index 2776828..605817c 100755 --- a/scripts/sweep_e4_pressured.sh +++ b/scripts/sweep_e4_pressured.sh @@ -54,6 +54,29 @@ log "OUTPUT=$OUTPUT" label=e4p_kvc_v2_d_to_p_sync_run1 log "=== [E4p] $label starting ===" +# Background GPU utilization sampler — every 1 s, all 4 GPUs, CSV output. +GPU_CSV="$OUTPUT/gpu_util.csv" +log "GPU sampling → $GPU_CSV (1 Hz, gpus 0-3)" +echo "timestamp_iso,gpu_index,util_pct,mem_used_MiB,mem_total_MiB,sm_clock_MHz,power_W,temperature_C" > "$GPU_CSV" +( + while true; do + ts_iso=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) + nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total,clocks.sm,power.draw,temperature.gpu \ + --format=csv,noheader,nounits 2>/dev/null \ + | sed -e "s/^/${ts_iso},/" -e 's/ //g' >> "$GPU_CSV" || true + sleep 1 + done +) & +GPU_SAMPLER_PID=$! +log "GPU sampler pid=$GPU_SAMPLER_PID" + +cleanup_gpu_sampler() { + kill -9 "$GPU_SAMPLER_PID" 2>/dev/null || true + wait "$GPU_SAMPLER_PID" 2>/dev/null || true + log "GPU sampler stopped (output: $GPU_CSV, $(wc -l < "$GPU_CSV") rows)" +} +trap cleanup_gpu_sampler EXIT INT TERM + uv run --no-sync python -m agentic_pd_hybrid.cli benchmark-live \ --trace "$TRACE" \ --output-root "$OUTPUT" \