#!/usr/bin/env bash

# CPU-only Frontier surface for the per-GPU-normalized Qwen3-30B Trace-PD case.
# This script is copied to the remote experiment output directory, never into
# the shared/dirty AITuner checkout.

set -euo pipefail

OUT="${1:?output root is required}"
RUNNER="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py"
FRONTIER="/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a"
REPLAYSERVE="/home/admin/cpfs/wjh/replayserve"
PYTHON_DEPS="/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1/lib/python3.12/site-packages"
PROFILE="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final"
ALLREDUCE="/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv"
BASE_PUBLIC="/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv"
BASE_PRIVATE="/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl"
MATERIALIZER="${OUT}/bin/materialize_qwen30_tp_normalized_trace.py"

printf '%s\n' "SIMULATOR_LAUNCH_ECHO host=dash0; gpu_allocation=0; model=Qwen3-30B-A3B; trace=trace-exact-v1/u0p01; requests=129; transform=t_prime=t/TP; TP={1,2,4}; per_gpu_offered_rate=0.215 req/s; surface=TP{1,2,4}xMNS{8,16,32,64}; Frontier=deadc4a321f0baaa534c6ebd17f974123733cdc2; profiles=profile-v4-trace-final; expected_wall=10-45m; output=${OUT}"
date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ

mkdir -p "${OUT}/provenance"
sha256sum "${MATERIALIZER}" "${BASH_SOURCE[0]}" "${RUNNER}" "${BASE_PUBLIC}" \
  "${BASE_PRIVATE}" "${PROFILE}/manifest.json" "${ALLREDUCE}" \
  > "${OUT}/provenance/input.sha256"
git -C "${FRONTIER}" rev-parse HEAD > "${OUT}/provenance/frontier.commit"
git -C /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1 rev-parse HEAD \
  > "${OUT}/provenance/aituner.commit"
printf '%s\n' "${PYTHON_DEPS}" > "${OUT}/provenance/python_deps_path"
/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1/bin/python -c \
  'import importlib.metadata as m; [print(f"{name}=={m.version(name)}") for name in ("plotly", "numpy", "pandas", "scikit-learn", "PyYAML")]' \
  > "${OUT}/provenance/python_deps_versions.txt"

for tp in 1 2 4; do
  /usr/bin/python3 "${MATERIALIZER}" \
    --base-public-csv "${BASE_PUBLIC}" \
    --base-private-jsonl "${BASE_PRIVATE}" \
    --tp "${tp}" \
    --output-root "${OUT}/traces/tp${tp}" \
    > "${OUT}/traces-tp${tp}.log"
done

for tp in 1 2 4; do
  for mns in 8 16 32 64; do
    cfg="tp${tp}_mns${mns}"
    cfgout="${OUT}/simulator-retry-deps/${cfg}"
    mkdir -p "${cfgout}"
    (
      set +e
      timeout --signal=TERM --kill-after=30s 2400 /usr/bin/python3 "${RUNNER}" \
        --frontier-source "${FRONTIER}" \
        --replayserve-root "${REPLAYSERVE}" \
        --profile-root "${PROFILE}" \
        --python-deps "${PYTHON_DEPS}" \
        --output-root "${cfgout}" \
        --trace "tp${tp}=${OUT}/traces/tp${tp}/public/frontier.csv" \
        --config "${cfg}" \
        --rate-contract trace-window \
        --prefix-caching \
        --cc-backend vidur \
        --allreduce-csv "${ALLREDUCE}" \
        --timeout-seconds 1800 \
        --predictor-training-job-threads 12 \
        --continue-on-failure \
        > "${cfgout}/launcher.stdout.log" 2> "${cfgout}/launcher.stderr.log"
      printf '%s\n' "$?" > "${cfgout}/launcher.exit_code"
      exit 0
    ) &
  done
done
wait

find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
  | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256"
date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
printf '%s\n' 'SIMULATOR_SURFACE_COMPLETE'