78 lines
3.7 KiB
Bash
78 lines
3.7 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# CPU-only Frontier surface for the per-GPU-normalized Qwen3-30B Trace-PD case.
|
|
# This script is copied to the remote experiment output directory, never into
|
|
# the shared/dirty AITuner checkout.
|
|
|
|
set -euo pipefail
|
|
|
|
OUT="${1:?output root is required}"
|
|
RUNNER="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py"
|
|
FRONTIER="/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a"
|
|
REPLAYSERVE="/home/admin/cpfs/wjh/replayserve"
|
|
PYTHON_DEPS="/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1/lib/python3.12/site-packages"
|
|
PROFILE="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final"
|
|
ALLREDUCE="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv"
|
|
BASE_PUBLIC="/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv"
|
|
BASE_PRIVATE="/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl"
|
|
MATERIALIZER="${OUT}/bin/materialize_qwen30_tp_normalized_trace.py"
|
|
|
|
printf '%s\n' "SIMULATOR_LAUNCH_ECHO host=dash0; gpu_allocation=0; model=Qwen3-30B-A3B; trace=trace-exact-v1/u0p01; requests=129; transform=t_prime=t/TP; TP={1,2,4}; per_gpu_offered_rate=0.215 req/s; surface=TP{1,2,4}xMNS{8,16,32,64}; Frontier=deadc4a321f0baaa534c6ebd17f974123733cdc2; profiles=profile-v4-trace-final; expected_wall=10-45m; output=${OUT}"
|
|
date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
|
|
mkdir -p "${OUT}/provenance"
|
|
sha256sum "${MATERIALIZER}" "${BASH_SOURCE[0]}" "${RUNNER}" "${BASE_PUBLIC}" \
|
|
"${BASE_PRIVATE}" "${PROFILE}/manifest.json" "${ALLREDUCE}" \
|
|
> "${OUT}/provenance/input.sha256"
|
|
git -C "${FRONTIER}" rev-parse HEAD > "${OUT}/provenance/frontier.commit"
|
|
git -C /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1 rev-parse HEAD \
|
|
> "${OUT}/provenance/aituner.commit"
|
|
printf '%s\n' "${PYTHON_DEPS}" > "${OUT}/provenance/python_deps_path"
|
|
/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1/bin/python -c \
|
|
'import importlib.metadata as m; [print(f"{name}=={m.version(name)}") for name in ("plotly", "numpy", "pandas", "scikit-learn", "PyYAML")]' \
|
|
> "${OUT}/provenance/python_deps_versions.txt"
|
|
|
|
for tp in 1 2 4; do
|
|
/usr/bin/python3 "${MATERIALIZER}" \
|
|
--base-public-csv "${BASE_PUBLIC}" \
|
|
--base-private-jsonl "${BASE_PRIVATE}" \
|
|
--tp "${tp}" \
|
|
--output-root "${OUT}/traces/tp${tp}" \
|
|
> "${OUT}/traces-tp${tp}.log"
|
|
done
|
|
|
|
for tp in 1 2 4; do
|
|
for mns in 8 16 32 64; do
|
|
cfg="tp${tp}_mns${mns}"
|
|
cfgout="${OUT}/simulator-retry-deps/${cfg}"
|
|
mkdir -p "${cfgout}"
|
|
(
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 2400 /usr/bin/python3 "${RUNNER}" \
|
|
--frontier-source "${FRONTIER}" \
|
|
--replayserve-root "${REPLAYSERVE}" \
|
|
--profile-root "${PROFILE}" \
|
|
--python-deps "${PYTHON_DEPS}" \
|
|
--output-root "${cfgout}" \
|
|
--trace "tp${tp}=${OUT}/traces/tp${tp}/public/frontier.csv" \
|
|
--config "${cfg}" \
|
|
--rate-contract trace-window \
|
|
--prefix-caching \
|
|
--cc-backend vidur \
|
|
--allreduce-csv "${ALLREDUCE}" \
|
|
--timeout-seconds 1800 \
|
|
--predictor-training-job-threads 12 \
|
|
--continue-on-failure \
|
|
> "${cfgout}/launcher.stdout.log" 2> "${cfgout}/launcher.stderr.log"
|
|
printf '%s\n' "$?" > "${cfgout}/launcher.exit_code"
|
|
exit 0
|
|
) &
|
|
done
|
|
done
|
|
wait
|
|
|
|
find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
|
|
| sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256"
|
|
date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
|
|
printf '%s\n' 'SIMULATOR_SURFACE_COMPLETE'
|