diff --git a/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v2.toml b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v2.toml new file mode 100644 index 0000000..6699647 --- /dev/null +++ b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v2.toml @@ -0,0 +1,33 @@ +version = 1 + +[[jobs]] +name = "workload-regime-sim-tp1-v2-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash1"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp2-v2-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash2"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-noprefix-v2-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash3"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-prefix-v2-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash4"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true" +artifacts = [] diff --git a/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v3.toml b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v3.toml new file mode 100644 index 0000000..280cfb5 --- /dev/null +++ b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_v3.toml @@ -0,0 +1,33 @@ +version = 1 + +[[jobs]] +name = "workload-regime-sim-tp1-v3-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash1"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp2-v3-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash2"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-noprefix-v3-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash3"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-prefix-v3-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash4"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true" +artifacts = [] diff --git a/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh index 056b0bc..b2af749 100644 --- a/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh +++ b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh @@ -34,8 +34,8 @@ for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \ done run_mode() { - local prefix_mode="$1" output_root prefix_flag - local -a traces configs command + local prefix_mode="$1" output_root prefix_flag family + local -a families traces configs command output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}" mkdir -p "${output_root}/provenance" @@ -45,48 +45,68 @@ run_mode() { prefix_flag="--no-prefix-caching" fi - mapfile -t traces < <( + mapfile -t families < <( jq -r --argjson prefix "${prefix_mode}" ' .cases | map(select(.prefix_caching == $prefix)) - | sort_by(.global_offered_request_rate, .family) + | map(.family) + | unique | .[] - | "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)" ' "${TRACES_ROOT}/manifest.json" ) - [[ "${#traces[@]}" -gt 0 ]] || { echo "ERROR: empty trace group" >&2; exit 1; } + [[ "${#families[@]}" -gt 0 ]] || { echo "ERROR: empty family group" >&2; exit 1; } for mns in 8 16 32 64; do configs+=(--config "tp${TP}_mns${mns}") done - for trace in "${traces[@]}"; do - command+=(--trace "${trace}") - done - { printf 'host=%s\n' "$(hostname)" printf 'tp=%s\n' "${TP}" printf 'prefix_mode=%s\n' "${prefix_mode}" - printf 'trace_count=%s\n' "${#traces[@]}" + printf 'family_count=%s\n' "${#families[@]}" git -C "${CHECKOUT}" rev-parse HEAD git -C "${FRONTIER_SOURCE}" rev-parse HEAD sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \ "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" } > "${output_root}/provenance/launch.txt" - PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \ - --frontier-source "${FRONTIER_SOURCE}" \ - --replayserve-root "${REPLAYSERVE_ROOT}" \ - --profile-root "${PROFILE_ROOT}" \ - --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ - --python-deps "${CUSTOM_PYTHON_DEPS}" \ - --output-root "${output_root}" \ - "${command[@]}" "${configs[@]}" \ - --rate-contract trace-window "${prefix_flag}" \ - --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \ - --timeout-seconds 3600 --predictor-training-job-threads 4 \ - --decode-cuda-graph-mode piecewise --align-real-graph-runtime \ - --store-stage-batch-ledger --resume --continue-on-failure + for family in "${families[@]}"; do + traces=() + command=() + mapfile -t traces < <( + jq -r --arg family "${family}" ' + .cases + | map(select(.family == $family)) + | sort_by(.global_offered_request_rate) + | .[] + | "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)" + ' "${TRACES_ROOT}/manifest.json" + ) + [[ "${#traces[@]}" -eq 5 ]] || { + echo "ERROR: expected five load anchors for ${family}" >&2 + exit 1 + } + for trace in "${traces[@]}"; do + command+=(--trace "${trace}") + done + printf 'SIMULATOR_FAMILY_START family=%s tp=%s prefix=%s\n' \ + "${family}" "${TP}" "${prefix_mode}" + PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \ + --frontier-source "${FRONTIER_SOURCE}" \ + --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" \ + --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ + --python-deps "${CUSTOM_PYTHON_DEPS}" \ + --output-root "${output_root}" \ + "${command[@]}" "${configs[@]}" \ + --rate-contract trace-window "${prefix_flag}" \ + --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \ + --timeout-seconds 3600 --predictor-training-job-threads 4 \ + --decode-cuda-graph-mode piecewise --align-real-graph-runtime \ + --store-stage-batch-ledger --resume --continue-on-failure + cp "${output_root}/frontier_trace_surface_frozen.json" \ + "${output_root}/frontier_trace_surface_${family}.json" + done } if [[ "${MODE}" == "both" ]]; then