44 lines
2.3 KiB
Bash
44 lines
2.3 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
Q30_ROOT="${Q30_ROOT:?Q30_ROOT is required}"
|
|
Q30_SESSION="${Q30_SESSION:-q30_fixed_pressure_20260719}"
|
|
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
|
|
PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}"
|
|
RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
|
|
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
|
|
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
|
|
|
mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}"
|
|
exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1
|
|
echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}"
|
|
date -u +SUPERVISOR_START_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
|
|
while tmux has-session -t "${Q30_SESSION}" 2>/dev/null; do
|
|
printf 'WAIT_Q30_UTC=%s results=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
"$(find "${Q30_ROOT}/real" -path '*/results/result.json' 2>/dev/null | wc -l)"
|
|
sleep 30
|
|
done
|
|
grep -q 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' "${Q30_ROOT}/controller.log" || {
|
|
echo 'ERROR: Q30 session ended without completion marker; refusing Q235 launch' >&2
|
|
exit 1
|
|
}
|
|
nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 > 16 {exit 1}'
|
|
|
|
mkdir -p "${OUTPUT_ROOT}/profile-smoke"
|
|
env CUDA_VISIBLE_DEVICES=0 PYTHONPATH="${FRONTIER_SOURCE}" \
|
|
"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/smoke_qwen235_v020_frontier_moe.py" \
|
|
--frontier-source "${FRONTIER_SOURCE}" \
|
|
--output "${OUTPUT_ROOT}/profile-smoke/moe.json"
|
|
|
|
OUTPUT_ROOT="${PROFILE_ROOT}" FRONTIER_SOURCE="${FRONTIER_SOURCE}" \
|
|
VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
|
|
bash "${RUNNER_DIR}/run_qwen235_v020_profiles.sh"
|
|
|
|
CAMPAIGN_ROOT="${OUTPUT_ROOT}" PROFILE_ROOT="${PROFILE_ROOT}/frozen" \
|
|
FRONTIER_SOURCE="${FRONTIER_SOURCE}" VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \
|
|
bash "${RUNNER_DIR}/run_qwen235_v020_campaign.sh"
|
|
date -u +SUPERVISOR_END_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
echo Q235_AFTER_Q30_COMPLETE
|