experiment: add parallel workload simulator sweep

runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py

@@ -107,6 +107,11 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--resume", action="store_true")
     parser.add_argument("--continue-on-failure", action="store_true")
+    parser.add_argument(
+        "--store-stage-batch-ledger",
+        action="store_true",
+        help="Persist Frontier's per-stage batch/state ledger for diagnosis.",
+    )
     return parser.parse_args()
 
 
@@ -315,6 +320,17 @@ def classify_frontier_failure(stderr: str) -> str:
     return "frontier_error"
 
 
+def configure_stage_batch_ledger(command: list[str], *, enabled: bool) -> list[str]:
+    if not enabled:
+        return command
+    disabled = "--no-metrics_config_store_frontier_stage_batch_ledger"
+    enabled_flag = "--metrics_config_store_frontier_stage_batch_ledger"
+    if command.count(disabled) != 1 or enabled_flag in command:
+        raise ValueError("unexpected stage-batch-ledger command contract")
+    command[command.index(disabled)] = enabled_flag
+    return command
+
+
 def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]:
     with path.open(newline="") as source:
         rows = list(csv.DictReader(source))
@@ -592,6 +608,9 @@ def main() -> None:
                     str(args.predictor_training_job_threads),
                 ]
             )
+            command = configure_stage_batch_ledger(
+                command, enabled=args.store_stage_batch_ledger
+            )
             if args.align_real_graph_runtime:
                 command.extend(
                     [
@@ -803,6 +822,7 @@ def main() -> None:
             "prefix_caching": args.prefix_caching,
             "arrival": "original_trace_timestamp_and_order",
             "input_output": "exact_source_values",
+            "store_stage_batch_ledger": args.store_stage_batch_ledger,
             "ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
             "tpot_slos_ms": TPOT_SLOS_MS,
             "primary_tpot_slo_ms": 150.0,

runs/frontier-workload-regime-taxonomy-v0/jobs_simulator.toml

@@ -0,0 +1,33 @@
+version = 1
+
+[[jobs]]
+name = "workload-regime-sim-tp1-20260720"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash1"]
+command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both"
+artifacts = []
+
+[[jobs]]
+name = "workload-regime-sim-tp2-20260720"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash2"]
+command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both"
+artifacts = []
+
+[[jobs]]
+name = "workload-regime-sim-tp4-noprefix-20260720"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash3"]
+command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false"
+artifacts = []
+
+[[jobs]]
+name = "workload-regime-sim-tp4-prefix-20260720"
+gpus = 1
+gpu_model = "H20"
+hosts = ["dash4"]
+command = "bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true"
+artifacts = []

runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+TP="${1:?usage: run_simulator_group.sh TP {false|true|both}}"
+MODE="${2:?usage: run_simulator_group.sh TP {false|true|both}}"
+
+case "${TP}" in
+  1|2|4) ;;
+  *) echo "ERROR: TP must be 1, 2, or 4" >&2; exit 2 ;;
+esac
+case "${MODE}" in
+  false|true|both) ;;
+  *) echo "ERROR: mode must be false, true, or both" >&2; exit 2 ;;
+esac
+
+CHECKOUT="${CHECKOUT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
+EXPERIMENT_ROOT="${EXPERIMENT_ROOT:-/home/admin/cpfs/wjh/aituner/workload-regime-20260720}"
+TRACES_ROOT="${TRACES_ROOT:-${EXPERIMENT_ROOT}/traces}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}"
+REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}"
+PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}"
+KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}"
+ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}"
+CUSTOM_PYTHON_DEPS="${CUSTOM_PYTHON_DEPS:-${EXPERIMENT_ROOT}/python-deps}"
+BASE_PYTHON_DEPS="${BASE_PYTHON_DEPS:-/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2/lib/python3.12/site-packages}"
+RUNNER="${CHECKOUT}/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py"
+
+for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \
+  "${REPLAYSERVE_ROOT}" "${PROFILE_ROOT}" "${KERNEL_PROFILE_ROOT}" \
+  "${ALLREDUCE_CSV}" "${CUSTOM_PYTHON_DEPS}" "${BASE_PYTHON_DEPS}" \
+  "${RUNNER}"; do
+  [[ -e "${path}" ]] || { echo "ERROR: missing ${path}" >&2; exit 1; }
+done
+
+run_mode() {
+  local prefix_mode="$1" output_root prefix_flag
+  local -a traces configs command
+  output_root="${EXPERIMENT_ROOT}/sim/tp${TP}-prefix-${prefix_mode}"
+  mkdir -p "${output_root}/provenance"
+
+  if [[ "${prefix_mode}" == "true" ]]; then
+    prefix_flag="--prefix-caching"
+  else
+    prefix_flag="--no-prefix-caching"
+  fi
+
+  mapfile -t traces < <(
+    jq -r --argjson prefix "${prefix_mode}" '
+      .cases
+      | map(select(.prefix_caching == $prefix))
+      | sort_by(.global_offered_request_rate, .family)
+      | .[]
+      | "\(.family)-rho\(.rho | tostring | gsub("\\."; "p"))=\(.public_csv)"
+    ' "${TRACES_ROOT}/manifest.json"
+  )
+  [[ "${#traces[@]}" -gt 0 ]] || { echo "ERROR: empty trace group" >&2; exit 1; }
+
+  for mns in 8 16 32 64; do
+    configs+=(--config "tp${TP}_mns${mns}")
+  done
+  for trace in "${traces[@]}"; do
+    command+=(--trace "${trace}")
+  done
+
+  {
+    printf 'host=%s\n' "$(hostname)"
+    printf 'tp=%s\n' "${TP}"
+    printf 'prefix_mode=%s\n' "${prefix_mode}"
+    printf 'trace_count=%s\n' "${#traces[@]}"
+    git -C "${CHECKOUT}" rev-parse HEAD
+    git -C "${FRONTIER_SOURCE}" rev-parse HEAD
+    sha256sum "${TRACES_ROOT}/manifest.json" "${PROFILE_ROOT}/manifest.json" \
+      "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}"
+  } > "${output_root}/provenance/launch.txt"
+
+  PYTHONPATH="${BASE_PYTHON_DEPS}" /usr/bin/python3 "${RUNNER}" \
+    --frontier-source "${FRONTIER_SOURCE}" \
+    --replayserve-root "${REPLAYSERVE_ROOT}" \
+    --profile-root "${PROFILE_ROOT}" \
+    --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \
+    --python-deps "${CUSTOM_PYTHON_DEPS}" \
+    --output-root "${output_root}" \
+    "${command[@]}" "${configs[@]}" \
+    --rate-contract trace-window "${prefix_flag}" \
+    --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \
+    --timeout-seconds 3600 --predictor-training-job-threads 4 \
+    --decode-cuda-graph-mode piecewise --align-real-graph-runtime \
+    --store-stage-batch-ledger --resume --continue-on-failure
+}
+
+if [[ "${MODE}" == "both" ]]; then
+  run_mode false
+  run_mode true
+else
+  run_mode "${MODE}"
+fi