fix: sweep simulator families independently

This commit is contained in:
2026-07-20 18:04:23 +08:00
parent f38e260639
commit f727cbcf76
3 changed files with 110 additions and 24 deletions

View File

@@ -0,0 +1,33 @@
version = 1
[[jobs]]
name = "workload-regime-sim-tp1-v2-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash1"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp2-v2-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash2"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp4-noprefix-v2-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash3"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp4-prefix-v2-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash4"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
artifacts = []

View File

@@ -0,0 +1,33 @@
version = 1
[[jobs]]
name = "workload-regime-sim-tp1-v3-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash1"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp2-v3-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash2"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp4-noprefix-v3-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash3"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
artifacts = []
[[jobs]]
name = "workload-regime-sim-tp4-prefix-v3-20260720"
gpus = 1
gpu_model = "H20"
hosts = ["dash4"]
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
artifacts = []

View File

@@ -34,8 +34,8 @@ for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \
done done
run_mode() { run_mode() {
local prefix_mode="$1" output_root prefix_flag local prefix_mode="$1" output_root prefix_flag family
local -a traces configs command local -a families traces configs command
output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}" output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}"
mkdir -p "${output_root}/provenance" mkdir -p "${output_root}/provenance"
@@ -45,35 +45,52 @@ run_mode() {
prefix_flag="--no-prefix-caching" prefix_flag="--no-prefix-caching"
fi fi
mapfile -t traces < <( mapfile -t families < <(
jq -r --argjson prefix "${prefix_mode}" ' jq -r --argjson prefix "${prefix_mode}" '
.cases .cases
| map(select(.prefix_caching == $prefix)) | map(select(.prefix_caching == $prefix))
| sort_by(.global_offered_request_rate, .family) | map(.family)
| unique
| .[] | .[]
| "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)"
' "${TRACES_ROOT}/manifest.json" ' "${TRACES_ROOT}/manifest.json"
) )
[[ "${#traces[@]}" -gt 0 ]] || { echo "ERROR: empty trace group" >&2; exit 1; } [[ "${#families[@]}" -gt 0 ]] || { echo "ERROR: empty family group" >&2; exit 1; }
for mns in 8 16 32 64; do for mns in 8 16 32 64; do
configs+=(--config "tp${TP}_mns${mns}") configs+=(--config "tp${TP}_mns${mns}")
done done
for trace in "${traces[@]}"; do
command+=(--trace "${trace}")
done
{ {
printf 'host=%s\n' "$(hostname)" printf 'host=%s\n' "$(hostname)"
printf 'tp=%s\n' "${TP}" printf 'tp=%s\n' "${TP}"
printf 'prefix_mode=%s\n' "${prefix_mode}" printf 'prefix_mode=%s\n' "${prefix_mode}"
printf 'trace_count=%s\n' "${#traces[@]}" printf 'family_count=%s\n' "${#families[@]}"
git -C "${CHECKOUT}" rev-parse HEAD git -C "${CHECKOUT}" rev-parse HEAD
git -C "${FRONTIER_SOURCE}" rev-parse HEAD git -C "${FRONTIER_SOURCE}" rev-parse HEAD
sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \ sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \
"${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}"
} > "${output_root}/provenance/launch.txt" } > "${output_root}/provenance/launch.txt"
for family in "${families[@]}"; do
traces=()
command=()
mapfile -t traces < <(
jq -r --arg family "${family}" '
.cases
| map(select(.family == $family))
| sort_by(.global_offered_request_rate)
| .[]
| "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)"
' "${TRACES_ROOT}/manifest.json"
)
[[ "${#traces[@]}" -eq 5 ]] || {
echo "ERROR: expected five load anchors for ${family}" >&2
exit 1
}
for trace in "${traces[@]}"; do
command+=(--trace "${trace}")
done
printf 'SIMULATOR_FAMILY_START family=%s tp=%s prefix=%s\n' \
"${family}" "${TP}" "${prefix_mode}"
PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \ PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \
--frontier-source "${FRONTIER_SOURCE}" \ --frontier-source "${FRONTIER_SOURCE}" \
--replayserve-root "${REPLAYSERVE_ROOT}" \ --replayserve-root "${REPLAYSERVE_ROOT}" \
@@ -87,6 +104,9 @@ run_mode() {
--timeout-seconds 3600 --predictor-training-job-threads 4 \ --timeout-seconds 3600 --predictor-training-job-threads 4 \
--decode-cuda-graph-mode piecewise --align-real-graph-runtime \ --decode-cuda-graph-mode piecewise --align-real-graph-runtime \
--store-stage-batch-ledger --resume --continue-on-failure --store-stage-batch-ledger --resume --continue-on-failure
cp "${output_root}/frontier_trace_surface_frozen.json" \
"${output_root}/frontier_trace_surface_${family}.json"
done
} }
if [[ "${MODE}" == "both" ]]; then if [[ "${MODE}" == "both" ]]; then