diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml new file mode 100644 index 0000000..9f1eed8 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-frontier-linear-smoke-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_frontier_linear_smoke.sh" +artifacts = ["artifacts/frontier-linear-smoke-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh new file mode 100644 index 0000000..1d9232e --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" +exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=qwen3-a3b-30b-moe runtime=vLLM-0.20.0+cu129 profiler=Frontier@d9cfeb6 operator=linear,norm,rope,embedding tp=1,2,4 tokens=8 precision=BF16 method=cuda_event output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum run_frontier_linear_smoke.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" + +cd "${FRONTIER_ROOT}" +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + --disable_ray \ + --num_gpus 1 \ + --output_dir "${OUTPUT_ROOT}/profiles" \ + --device h20 \ + --models qwen3-a3b-30b-moe \ + --num_tensor_parallel_workers 1 2 4 \ + --num_tokens_list 8 \ + --profile_method cuda_event \ + --precision BF16 \ + --is_moe \ + --yes + +find "${OUTPUT_ROOT}/profiles" -type f -size +0c -print \ + > "${OUTPUT_ROOT}/provenance/profile-files.txt" +test -s "${OUTPUT_ROOT}/provenance/profile-files.txt" +find "${OUTPUT_ROOT}/profiles" -type f -print0 | sort -z | xargs -0 sha256sum \ + > "${OUTPUT_ROOT}/profiles.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FRONTIER_LINEAR_SMOKE_COMPLETE"