#!/usr/bin/env bash

set -euo pipefail

Q30_ROOT="${Q30_ROOT:?Q30_ROOT is required}"
Q30_SESSION="${Q30_SESSION:-q30_fixed_pressure_20260719}"
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"

# FlashInfer's TensorRT-LLM MoE runtime invokes `nvcc` by name when it
# autotunes a previously unseen FP8 grouped-GEMM shape.
export PATH="/usr/local/cuda/bin:${PATH}"
export FRONTIER_VLLM_MODEL_TYPE=qwen3_moe
export VLLM_KV_CACHE_LAYOUT=NHD
command -v nvcc >/dev/null

mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}"
exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1
echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}"
date -u +SUPERVISOR_START_UTC=%Y-%m-%dT%H:%M:%SZ

while tmux has-session -t "${Q30_SESSION}" 2>/dev/null; do
  printf 'WAIT_Q30_UTC=%s results=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
    "$(find "${Q30_ROOT}/real" -path '*/results/result.json' 2>/dev/null | wc -l)"
  sleep 30
done
grep -q 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' "${Q30_ROOT}/controller.log" || {
  echo 'ERROR: Q30 session ended without completion marker; refusing Q235 launch' >&2
  exit 1
}
nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 > 16 {exit 1}'

mkdir -p "${OUTPUT_ROOT}/profile-smoke"
env CUDA_VISIBLE_DEVICES=0 PYTHONPATH="${FRONTIER_SOURCE}" \
  "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/smoke_qwen235_v020_frontier_moe.py" \
    --frontier-source "${FRONTIER_SOURCE}" \
    --output "${OUTPUT_ROOT}/profile-smoke/moe.json"

OUTPUT_ROOT="${PROFILE_ROOT}" FRONTIER_SOURCE="${FRONTIER_SOURCE}" \
  VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
  bash "${RUNNER_DIR}/run_qwen235_v020_profiles.sh"

CAMPAIGN_ROOT="${OUTPUT_ROOT}" PROFILE_ROOT="${PROFILE_ROOT}/frozen" \
  FRONTIER_SOURCE="${FRONTIER_SOURCE}" VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
  bash "${RUNNER_DIR}/run_qwen235_v020_campaign.sh"
date -u +SUPERVISOR_END_UTC=%Y-%m-%dT%H:%M:%SZ
echo Q235_AFTER_Q30_COMPLETE