fix: isolate simulator predictor cache
This commit is contained in:
@@ -611,6 +611,12 @@ def main() -> None:
|
|||||||
command = configure_stage_batch_ledger(
|
command = configure_stage_batch_ledger(
|
||||||
command, enabled=args.store_stage_batch_ledger
|
command, enabled=args.store_stage_batch_ledger
|
||||||
)
|
)
|
||||||
|
command.extend(
|
||||||
|
[
|
||||||
|
"--metrics_config_cache_dir",
|
||||||
|
str(args.output_root / "cache"),
|
||||||
|
]
|
||||||
|
)
|
||||||
if args.align_real_graph_runtime:
|
if args.align_real_graph_runtime:
|
||||||
command.extend(
|
command.extend(
|
||||||
[
|
[
|
||||||
@@ -640,7 +646,7 @@ def main() -> None:
|
|||||||
)
|
)
|
||||||
fallback_evidence = (
|
fallback_evidence = (
|
||||||
collective_fallback_evidence(run_dir)
|
collective_fallback_evidence(run_dir)
|
||||||
if args.cc_backend == "vidur"
|
if args.cc_backend == "vidur" and config.tp > 1
|
||||||
else []
|
else []
|
||||||
)
|
)
|
||||||
if completed.returncode == 0 and fallback_evidence:
|
if completed.returncode == 0 and fallback_evidence:
|
||||||
@@ -823,6 +829,7 @@ def main() -> None:
|
|||||||
"arrival": "original_trace_timestamp_and_order",
|
"arrival": "original_trace_timestamp_and_order",
|
||||||
"input_output": "exact_source_values",
|
"input_output": "exact_source_values",
|
||||||
"store_stage_batch_ledger": args.store_stage_batch_ledger,
|
"store_stage_batch_ledger": args.store_stage_batch_ledger,
|
||||||
|
"predictor_cache_root": str(args.output_root / "cache"),
|
||||||
"ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
|
"ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
|
||||||
"tpot_slos_ms": TPOT_SLOS_MS,
|
"tpot_slos_ms": TPOT_SLOS_MS,
|
||||||
"primary_tpot_slo_ms": 150.0,
|
"primary_tpot_slo_ms": 150.0,
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
version = 1
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
name = "workload-regime-sim-tp1-v4-20260720"
|
||||||
|
gpus = 1
|
||||||
|
gpu_model = "H20"
|
||||||
|
hosts = ["dash1"]
|
||||||
|
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
|
||||||
|
artifacts = []
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
name = "workload-regime-sim-tp2-v4-20260720"
|
||||||
|
gpus = 1
|
||||||
|
gpu_model = "H20"
|
||||||
|
hosts = ["dash2"]
|
||||||
|
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
|
||||||
|
artifacts = []
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
name = "workload-regime-sim-tp4-noprefix-v4-20260720"
|
||||||
|
gpus = 1
|
||||||
|
gpu_model = "H20"
|
||||||
|
hosts = ["dash3"]
|
||||||
|
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
|
||||||
|
artifacts = []
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
name = "workload-regime-sim-tp4-prefix-v4-20260720"
|
||||||
|
gpus = 1
|
||||||
|
gpu_model = "H20"
|
||||||
|
hosts = ["dash4"]
|
||||||
|
command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
|
||||||
|
artifacts = []
|
||||||
@@ -22,8 +22,9 @@ REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
|
|||||||
PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}"
|
PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}"
|
||||||
KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}"
|
KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}"
|
||||||
ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}"
|
ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}"
|
||||||
CUSTOM_PYTHON_DEPS="${CUSTOM_PYTHON_DEPS:-${EXPERIMENT_ROOT}/python-deps}"
|
CUSTOM_PYTHON_DEPS="${CUSTOM_PYTHON_DEPS:-${EXPERIMENT_ROOT}/python-deps-v2}"
|
||||||
BASE_PYTHON_DEPS="${BASE_PYTHON_DEPS:-/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2/lib/python3.12/site-packages}"
|
BASE_PYTHON_DEPS="${BASE_PYTHON_DEPS:-/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2/lib/python3.12/site-packages}"
|
||||||
|
SIM_GENERATION="${SIM_GENERATION:-v4}"
|
||||||
RUNNER="${CHECKOUT}/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py"
|
RUNNER="${CHECKOUT}/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py"
|
||||||
|
|
||||||
for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \
|
for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \
|
||||||
@@ -36,7 +37,7 @@ done
|
|||||||
run_mode() {
|
run_mode() {
|
||||||
local prefix_mode="$1" output_root prefix_flag family
|
local prefix_mode="$1" output_root prefix_flag family
|
||||||
local -a families traces configs command
|
local -a families traces configs command
|
||||||
output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}"
|
output_root="${EXPERIMENT_ROOT}/sim-${SIM_GENERATION}/tp${TP}-prefix-${prefix_mode}"
|
||||||
mkdir -p "${output_root}/provenance"
|
mkdir -p "${output_root}/provenance"
|
||||||
|
|
||||||
if [[ "${prefix_mode}" == "true" ]]; then
|
if [[ "${prefix_mode}" == "true" ]]; then
|
||||||
|
|||||||
Reference in New Issue
Block a user