From 79e9870975b662a8f037e1e42c2317e0723010b0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:54:32 +0800 Subject: [PATCH] Defer Qwen235 campaign until Qwen30 completion --- .../run_q235_after_q30.sh | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh diff --git a/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh new file mode 100644 index 0000000..a3e6a05 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +Q30_ROOT="${Q30_ROOT:?Q30_ROOT is required}" +Q30_SESSION="${Q30_SESSION:-q30_fixed_pressure_20260719}" +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" + +mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}" +exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1 +echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}" +date -u +SUPERVISOR_START_UTC=%Y-%m-%dT%H:%M:%SZ + +while tmux has-session -t "${Q30_SESSION}" 2>/dev/null; do + printf 'WAIT_Q30_UTC=%s results=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "$(find "${Q30_ROOT}/real" -path '*/results/result.json' 2>/dev/null | wc -l)" + sleep 30 +done +grep -q 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' "${Q30_ROOT}/controller.log" || { + echo 'ERROR: Q30 session ended without completion marker; refusing Q235 launch' >&2 + exit 1 +} +nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 > 16 {exit 1}' + +mkdir -p "${OUTPUT_ROOT}/profile-smoke" +env CUDA_VISIBLE_DEVICES=0 PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/smoke_qwen235_v020_frontier_moe.py" \ + --frontier-source "${FRONTIER_SOURCE}" \ + --output "${OUTPUT_ROOT}/profile-smoke/moe.json" + +OUTPUT_ROOT="${PROFILE_ROOT}" FRONTIER_SOURCE="${FRONTIER_SOURCE}" \ + VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \ + bash "${RUNNER_DIR}/run_qwen235_v020_profiles.sh" + +CAMPAIGN_ROOT="${OUTPUT_ROOT}" PROFILE_ROOT="${PROFILE_ROOT}/frozen" \ + FRONTIER_SOURCE="${FRONTIER_SOURCE}" VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \ + bash "${RUNNER_DIR}/run_qwen235_v020_campaign.sh" +date -u +SUPERVISOR_END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_AFTER_Q30_COMPLETE