Align Frontier piecewise graph profiles
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}"
|
||||
RUN_DIR="$(pwd -P)"
|
||||
PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}"
|
||||
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
||||
VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}"
|
||||
FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}"
|
||||
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
||||
TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128)
|
||||
export MODEL_ROOT
|
||||
|
||||
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles"
|
||||
exec > >(tee -a "${OUTPUT_ROOT}/logs/linear.log") 2>&1
|
||||
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?a fleet-allocated GPU is required}"
|
||||
if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
|
||||
echo "expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} role=linear-kernel-only tp=1,2,4 tokens='${TOKENS[*]}' method=Frontier-RecordFunctionTracer output=${OUTPUT_ROOT} expected_wall=15-35m expected_gpu_cap=1.0_H20h"
|
||||
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
||||
test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "deadc4a321f0baaa534c6ebd17f974123733cdc2"
|
||||
test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1"
|
||||
git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit"
|
||||
git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
|
||||
sha256sum "${PROFILE_DIR}/frontier_vllm020_compat.py" "${RUN_DIR}/run_graph_kernel_only_linear.sh" > "${OUTPUT_ROOT}/provenance/source.sha256"
|
||||
|
||||
cd "${FRONTIER_SOURCE}"
|
||||
timeout --signal=TERM --kill-after=30s 2400 \
|
||||
"${VENV_ROOT}/bin/python" "${PROFILE_DIR}/frontier_vllm020_compat.py" \
|
||||
--disable_ray --num_gpus 1 --output_dir "${OUTPUT_ROOT}/profiles" \
|
||||
--device h20 --models qwen3-a3b-30b-moe \
|
||||
--num_tensor_parallel_workers 1 2 4 --max_tokens 128 \
|
||||
--num_tokens_list "${TOKENS[@]}" --profile_method record_function \
|
||||
--precision BF16 --is_moe --yes
|
||||
|
||||
find "${OUTPUT_ROOT}/profiles" -name linear_op_kernel_only.csv -type f -size +0c -print -quit > "${OUTPUT_ROOT}/provenance/linear-path.txt"
|
||||
test -s "${OUTPUT_ROOT}/provenance/linear-path.txt"
|
||||
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
|
||||
echo "GRAPH_KERNEL_ONLY_LINEAR_COMPLETE"
|
||||
Reference in New Issue
Block a user