#!/usr/bin/env bash

set -euo pipefail

OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}"
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}"
VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}"
RUN_DIR="$(pwd -P)"
TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192)
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles"
exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1

IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}"
if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then
  echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2
  exit 1
fi

echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=qwen3-a3b-30b-moe runtime=vLLM-0.20.0+cu129 profiler=Frontier@d9cfeb6 operator=linear,norm,rope,embedding tp=1,2,4 tokens=${TOKENS[*]} precision=BF16 method=cuda_event output=${OUTPUT_ROOT} expected_wall=3-10m hard_wall=1500s hard_gpu_cap=0.42_H20h"
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader

test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e"
test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1"
git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit"
git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit"
sha256sum run_frontier_linear_full.sh frontier_vllm020_compat.py \
  > "${OUTPUT_ROOT}/provenance/source.sha256"
uv pip freeze --python "${VENV_ROOT}/bin/python" \
  > "${OUTPUT_ROOT}/provenance/pip-freeze.txt"
nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \
  --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv"
printf '%s\n' "${TOKENS[@]}" > "${OUTPUT_ROOT}/provenance/tokens.txt"

cd "${FRONTIER_ROOT}"
timeout --signal=TERM --kill-after=30s 1380 \
  "${VENV_ROOT}/bin/python" "${RUN_DIR}/frontier_vllm020_compat.py" \
  --disable_ray \
  --num_gpus 1 \
  --output_dir "${OUTPUT_ROOT}/profiles" \
  --device h20 \
  --models qwen3-a3b-30b-moe \
  --num_tensor_parallel_workers 1 2 4 \
  --num_tokens_list "${TOKENS[@]}" \
  --profile_method cuda_event \
  --precision BF16 \
  --is_moe \
  --yes

find "${OUTPUT_ROOT}/profiles" -type f -size +0c -print \
  > "${OUTPUT_ROOT}/provenance/profile-files.txt"
test -s "${OUTPUT_ROOT}/provenance/profile-files.txt"
find "${OUTPUT_ROOT}/profiles" -type f -print0 | sort -z | xargs -0 sha256sum \
  > "${OUTPUT_ROOT}/profiles.sha256"
date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ"
echo "FRONTIER_LINEAR_FULL_COMPLETE"