diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 5590ed8..9c869d8 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -107,6 +107,11 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--resume", action="store_true") parser.add_argument("--continue-on-failure", action="store_true") + parser.add_argument( + "--store-stage-batch-ledger", + action="store_true", + help="Persist Frontier's per-stage batch/state ledger for diagnosis.", + ) return parser.parse_args() @@ -315,6 +320,17 @@ def classify_frontier_failure(stderr: str) -> str: return "frontier_error" +def configure_stage_batch_ledger(command: list[str], *, enabled: bool) -> list[str]: + if not enabled: + return command + disabled = "--no-metrics_config_store_frontier_stage_batch_ledger" + enabled_flag = "--metrics_config_store_frontier_stage_batch_ledger" + if command.count(disabled) != 1 or enabled_flag in command: + raise ValueError("unexpected stage-batch-ledger command contract") + command[command.index(disabled)] = enabled_flag + return command + + def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: with path.open(newline="") as source: rows = list(csv.DictReader(source)) @@ -592,6 +608,9 @@ def main() -> None: str(args.predictor_training_job_threads), ] ) + command = configure_stage_batch_ledger( + command, enabled=args.store_stage_batch_ledger + ) if args.align_real_graph_runtime: command.extend( [ @@ -803,6 +822,7 @@ def main() -> None: "prefix_caching": args.prefix_caching, "arrival": "original_trace_timestamp_and_order", "input_output": "exact_source_values", + "store_stage_batch_ledger": args.store_stage_batch_ledger, "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", "tpot_slos_ms": TPOT_SLOS_MS, "primary_tpot_slo_ms": 150.0, diff --git a/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator.toml b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator.toml new file mode 100644 index 0000000..783afac --- /dev/null +++ b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator.toml @@ -0,0 +1,33 @@ +version = 1 + +[[jobs]] +name = "workload-regime-sim-tp1-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash1"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp2-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash2"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-noprefix-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash3"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false" +artifacts = [] + +[[jobs]] +name = "workload-regime-sim-tp4-prefix-20260720" +gpus = 1 +gpu_model = "H20" +hosts = ["dash4"] +command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true" +artifacts = [] diff --git a/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh new file mode 100644 index 0000000..c77c051 --- /dev/null +++ b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${1:?usage: run_simulator_group.sh TP {false|true|both}}" +MODE="${2:?usage: run_simulator_group.sh TP {false|true|both}}" + +case "${TP}" in + 1|2|4) ;; + *) echo "ERROR: TP must be 1, 2, or 4" >&2; exit 2 ;; +esac +case "${MODE}" in + false|true|both) ;; + *) echo "ERROR: mode must be false, true, or both" >&2; exit 2 ;; +esac + +CHECKOUT="${CHECKOUT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +EXPERIMENT_ROOT="${EXPERIMENT_ROOT:-/home/admin/cpfs/wjh/aituner/workload-regime-20260720}" +TRACES_ROOT="${TRACES_ROOT:-${EXPERIMENT_ROOT}/traces}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}" +KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}" +CUSTOM_PYTHON_DEPS="${CUSTOM_PYTHON_DEPS:-${EXPERIMENT_ROOT}/python-deps}" +BASE_PYTHON_DEPS="${BASE_PYTHON_DEPS:-/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2/lib/python3.12/site-packages}" +RUNNER="${CHECKOUT}/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py" + +for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \ + "${REPLAYSERVE_ROOT}" "${PROFILE_ROOT}" "${KERNEL_PROFILE_ROOT}" \ + "${ALLREDUCE_CSV}" "${CUSTOM_PYTHON_DEPS}" "${BASE_PYTHON_DEPS}" \ + "${RUNNER}"; do + [[ -e "${path}" ]] || { echo "ERROR: missing ${path}" >&2; exit 1; } +done + +run_mode() { + local prefix_mode="$1" output_root prefix_flag + local -a traces configs command + output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}" + mkdir -p "${output_root}/provenance" + + if [[ "${prefix_mode}" == "true" ]]; then + prefix_flag="--prefix-caching" + else + prefix_flag="--no-prefix-caching" + fi + + mapfile -t traces < <( + jq -r --argjson prefix "${prefix_mode}" ' + .cases + | map(select(.prefix_caching == $prefix)) + | sort_by(.global_offered_request_rate, .family) + | .[] + | "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)" + ' "${TRACES_ROOT}/manifest.json" + ) + [[ "${#traces[@]}" -gt 0 ]] || { echo "ERROR: empty trace group" >&2; exit 1; } + + for mns in 8 16 32 64; do + configs+=(--config "tp${TP}_mns${mns}") + done + for trace in "${traces[@]}"; do + command+=(--trace "${trace}") + done + + { + printf 'host=%s\n' "$(hostname)" + printf 'tp=%s\n' "${TP}" + printf 'prefix_mode=%s\n' "${prefix_mode}" + printf 'trace_count=%s\n' "${#traces[@]}" + git -C "${CHECKOUT}" rev-parse HEAD + git -C "${FRONTIER_SOURCE}" rev-parse HEAD + sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \ + "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" + } > "${output_root}/provenance/launch.txt" + + PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \ + --frontier-source "${FRONTIER_SOURCE}" \ + --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" \ + --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ + --python-deps "${CUSTOM_PYTHON_DEPS}" \ + --output-root "${output_root}" \ + "${command[@]}" "${configs[@]}" \ + --rate-contract trace-window "${prefix_flag}" \ + --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \ + --timeout-seconds 3600 --predictor-training-job-threads 4 \ + --decode-cuda-graph-mode piecewise --align-real-graph-runtime \ + --store-stage-batch-ledger --resume --continue-on-failure +} + +if [[ "${MODE}" == "both" ]]; then + run_mode false + run_mode true +else + run_mode "${MODE}" +fi