fix: sweep simulator families independently
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp1-v2-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash1"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp2-v2-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash2"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp4-noprefix-v2-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash3"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp4-prefix-v2-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash4"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
|
||||
artifacts = []
|
||||
@@ -0,0 +1,33 @@
|
||||
version = 1
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp1-v3-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash1"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp2-v3-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash2"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp4-noprefix-v3-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash3"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
|
||||
artifacts = []
|
||||
|
||||
[[jobs]]
|
||||
name = "workload-regime-sim-tp4-prefix-v3-20260720"
|
||||
gpus = 1
|
||||
gpu_model = "H20"
|
||||
hosts = ["dash4"]
|
||||
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
|
||||
artifacts = []
|
||||
@@ -34,8 +34,8 @@ for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \
|
||||
done
|
||||
|
||||
run_mode() {
|
||||
local prefix_mode="$1" output_root prefix_flag
|
||||
local -a traces configs command
|
||||
local prefix_mode="$1" output_root prefix_flag family
|
||||
local -a families traces configs command
|
||||
output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}"
|
||||
mkdir -p "${output_root}/provenance"
|
||||
|
||||
@@ -45,35 +45,52 @@ run_mode() {
|
||||
prefix_flag="--no-prefix-caching"
|
||||
fi
|
||||
|
||||
mapfile -t traces < <(
|
||||
mapfile -t families < <(
|
||||
jq -r --argjson prefix "${prefix_mode}" '
|
||||
.cases
|
||||
| map(select(.prefix_caching == $prefix))
|
||||
| sort_by(.global_offered_request_rate, .family)
|
||||
| map(.family)
|
||||
| unique
|
||||
| .[]
|
||||
| "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)"
|
||||
' "${TRACES_ROOT}/manifest.json"
|
||||
)
|
||||
[[ "${#traces[@]}" -gt 0 ]] || { echo "ERROR: empty trace group" >&2; exit 1; }
|
||||
[[ "${#families[@]}" -gt 0 ]] || { echo "ERROR: empty family group" >&2; exit 1; }
|
||||
|
||||
for mns in 8 16 32 64; do
|
||||
configs+=(--config "tp${TP}_mns${mns}")
|
||||
done
|
||||
for trace in "${traces[@]}"; do
|
||||
command+=(--trace "${trace}")
|
||||
done
|
||||
|
||||
{
|
||||
printf 'host=%s\n' "$(hostname)"
|
||||
printf 'tp=%s\n' "${TP}"
|
||||
printf 'prefix_mode=%s\n' "${prefix_mode}"
|
||||
printf 'trace_count=%s\n' "${#traces[@]}"
|
||||
printf 'family_count=%s\n' "${#families[@]}"
|
||||
git -C "${CHECKOUT}" rev-parse HEAD
|
||||
git -C "${FRONTIER_SOURCE}" rev-parse HEAD
|
||||
sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \
|
||||
"${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}"
|
||||
} > "${output_root}/provenance/launch.txt"
|
||||
|
||||
for family in "${families[@]}"; do
|
||||
traces=()
|
||||
command=()
|
||||
mapfile -t traces < <(
|
||||
jq -r --arg family "${family}" '
|
||||
.cases
|
||||
| map(select(.family == $family))
|
||||
| sort_by(.global_offered_request_rate)
|
||||
| .[]
|
||||
| "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)"
|
||||
' "${TRACES_ROOT}/manifest.json"
|
||||
)
|
||||
[[ "${#traces[@]}" -eq 5 ]] || {
|
||||
echo "ERROR: expected five load anchors for ${family}" >&2
|
||||
exit 1
|
||||
}
|
||||
for trace in "${traces[@]}"; do
|
||||
command+=(--trace "${trace}")
|
||||
done
|
||||
printf 'SIMULATOR_FAMILY_START family=%s tp=%s prefix=%s\n' \
|
||||
"${family}" "${TP}" "${prefix_mode}"
|
||||
PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \
|
||||
--frontier-source "${FRONTIER_SOURCE}" \
|
||||
--replayserve-root "${REPLAYSERVE_ROOT}" \
|
||||
@@ -87,6 +104,9 @@ run_mode() {
|
||||
--timeout-seconds 3600 --predictor-training-job-threads 4 \
|
||||
--decode-cuda-graph-mode piecewise --align-real-graph-runtime \
|
||||
--store-stage-batch-ledger --resume --continue-on-failure
|
||||
cp "${output_root}/frontier_trace_surface_frozen.json" \
|
||||
"${output_root}/frontier_trace_surface_${family}.json"
|
||||
done
|
||||
}
|
||||
|
||||
if [[ "${MODE}" == "both" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user