#!/usr/bin/env bash
# Run the PP=1/2/4 sweep with xserv and llama.cpp CONCURRENTLY on disjoint GPU
# groups: xserv (--pp) on GPUs 0..N-1, llama.cpp (-sm layer) on GPUs 4..4+N-1.
# The 8x5090 box is grouped 0-3 / 4-7 (PHB intra-group), so each engine's P2P
# stays intra-group and the two engines never contend for a GPU.
#
# xserv splits layers across N GPUs and hands off hidden states via NCCL P2P;
# llama.cpp's default `-sm layer` does the analogous layer-wise split.
#
# Run from the repo root on the GPU host. Produces bench-out/pp{1,2,4}-{xserv,llama}.

set -u
MODEL="${MODEL:-/opt/wjh/models/qwen3-8b}"
GGUF="${GGUF:-/opt/wjh/models/qwen3-8b/qwen3-8b-bf16.gguf}"
LIMIT="${LIMIT:-20}"
MAXSEQ="${MAXSEQ:-2048}"
PPS="${PPS:-1 2 4}"
TASKS="${TASKS:-gsm8k}"

for PP in $PPS; do
    LD=$(seq -s, 4 $((3 + PP)))   # llama GPUs: 4 / 4,5 / 4,5,6,7
    echo "##### PP=$PP  (xserv GPU 0..$((PP-1))  ||  llama GPU $LD) #####"
    rm -rf "bench-out/pp$PP-xserv" "bench-out/pp$PP-llama"

    python3 -u -m tools.bench.runner --systems xserv --pp "$PP" \
        --xserv-bin ./target/release/xserv-server --xserv-model "$MODEL" \
        --suite quality --quality-tasks "$TASKS" --quality-limit "$LIMIT" \
        --max-batch 1 --max-seq-len "$MAXSEQ" \
        --out-dir "bench-out/pp$PP-xserv" > "/tmp/pp$PP-xserv.log" 2>&1 &
    XP=$!

    python3 -u -m tools.bench.runner --systems llama.cpp --pp "$PP" --llama-devices "$LD" \
        --llama-bin third_party/llama.cpp/build/bin/llama-server --llama-gguf "$GGUF" \
        --suite quality --quality-tasks "$TASKS" --quality-limit "$LIMIT" \
        --max-batch 1 --max-seq-len "$MAXSEQ" \
        --out-dir "bench-out/pp$PP-llama" > "/tmp/pp$PP-llama.log" 2>&1 &
    LP=$!

    wait "$XP" "$LP"
    echo "PP=$PP done"
done
echo ALL_DONE