From e651ecc9230e9f580327e43dd096f757f79e133d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Mon, 20 Jul 2026 18:51:31 +0800 Subject: [PATCH] experiment: reuse predictors across load contracts --- .../frontier-workload-regime-taxonomy-20260720.md | 1 + .../run_frontier_qwen30_exact_trace_surface.py | 12 +++++++++--- .../jobs_simulator_per_gpu_v1.toml | 8 ++++---- .../run_simulator_group.sh | 9 ++++++++- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.research/experiments/frontier-workload-regime-taxonomy-20260720.md b/.research/experiments/frontier-workload-regime-taxonomy-20260720.md index 005dd6e..e198ad4 100644 --- a/.research/experiments/frontier-workload-regime-taxonomy-20260720.md +++ b/.research/experiments/frontier-workload-regime-taxonomy-20260720.md @@ -176,6 +176,7 @@ The accompanying mock figure is schematic, not data. Panel A shows the state kne - Real-runtime gate: a stock vLLM 0.20.0 environment passed import/H20 checks but used CUDA 13.0, so it is excluded from comparison with the historical CUDA 12.9 baseline. A separate official vLLM `0.20.0+cu129` environment is being materialized from the shared package cache before any real pilot is launched. - Load-contract correction: the original Fixed-PD surface held request rate per GPU constant, so global arrival rate scaled with TP. The v4 sweep holds global arrival rate constant and is retained as the control that isolates service-topology changes. A matched per-GPU sweep is now required to reproduce the original closed-loop intervention: TP1/TP2/TP4 receive `1x/2x/4x` global arrival rate at the same per-GPU `rho`. - Per-GPU low-load materialization: 105 cases = W0--W6 × `rho {0.0025,0.005,0.01,0.02,0.05}` × TP `{1,2,4}` were generated under `traces-per-gpu-low`. Audit passed 105 unique paths, 129 public/private rows per case, digests, arrival alignment, and exact `global_rate / TP = per_gpu_rate`. W0 `rho=0.01` is 0.239375 req/s/GPU, bracketing the original 0.215 req/s/GPU Fixed-PD point with `rho=0.005`. +- The per-GPU sweep writes to a separate `sim-per-gpu-v1` result root but reuses the completed v4 predictor cache for the same TP/prefix/config. Predictor cache provenance is explicit in every surface manifest; workload results and state ledgers are never shared. - A first materialization attempt rounded both `rho=0.005` and `rho=0.01` to the same `rho0p01` directory. Digest validation stopped before simulator launch; the invalid directories were retained with an `invalid-rho-label-collision` suffix. The label function now preserves up to 12 significant digits and has a regression test. Current decision: finish the v4 fixed-global-rate control, then reuse its trained predictors for the low-load per-GPU sweep before selecting discriminating real-machine pilot points. No real latency result from vLLM 0.20.2 will be compared with the historical vLLM 0.20.0 baseline until the runtime-version gate is resolved. diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 57c0346..317dc35 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -67,6 +67,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--kernel-profile-root", type=Path) parser.add_argument("--python-deps", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument( + "--predictor-cache-root", + type=Path, + help="Reuse a validated predictor cache outside the output root.", + ) parser.add_argument( "--trace", action="append", @@ -558,9 +563,10 @@ def main() -> None: ) config_results = [] + predictor_cache_root = args.predictor_cache_root or args.output_root / "cache" for config in selected: loads = [] - config_knobs = BASE.knobs(config, paths, args.output_root / "cache") + config_knobs = BASE.knobs(config, paths, predictor_cache_root) config_knobs["enable_prefix_caching"] = args.prefix_caching config_knobs["prediction_max_tokens_per_request"] = 40960 config_knobs["decode_cuda_graph_mode"] = args.decode_cuda_graph_mode @@ -614,7 +620,7 @@ def main() -> None: command.extend( [ "--metrics_config_cache_dir", - str(args.output_root / "cache"), + str(predictor_cache_root), ] ) if args.align_real_graph_runtime: @@ -829,7 +835,7 @@ def main() -> None: "arrival": "original_trace_timestamp_and_order", "input_output": "exact_source_values", "store_stage_batch_ledger": args.store_stage_batch_ledger, - "predictor_cache_root": str(args.output_root / "cache"), + "predictor_cache_root": str(predictor_cache_root), "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", "tpot_slos_ms": TPOT_SLOS_MS, "primary_tpot_slo_ms": 150.0, diff --git a/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_per_gpu_v1.toml b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_per_gpu_v1.toml index 120f36b..f642006 100644 --- a/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_per_gpu_v1.toml +++ b/runs/frontier-workload-regime-taxonomy-v0/jobs_simulator_per_gpu_v1.toml @@ -5,7 +5,7 @@ name = "workload-regime-per-gpu-sim-tp1-v1-20260720" gpus = 1 gpu_model = "H20" hosts = ["dash1"] -command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both" +command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 PREDICTOR_CACHE_GENERATION=v4 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 1 both" artifacts = [] [[jobs]] @@ -13,7 +13,7 @@ name = "workload-regime-per-gpu-sim-tp2-v1-20260720" gpus = 1 gpu_model = "H20" hosts = ["dash2"] -command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both" +command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 PREDICTOR_CACHE_GENERATION=v4 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 2 both" artifacts = [] [[jobs]] @@ -21,7 +21,7 @@ name = "workload-regime-per-gpu-sim-tp4-noprefix-v1-20260720" gpus = 1 gpu_model = "H20" hosts = ["dash3"] -command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false" +command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 PREDICTOR_CACHE_GENERATION=v4 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 false" artifacts = [] [[jobs]] @@ -29,5 +29,5 @@ name = "workload-regime-per-gpu-sim-tp4-prefix-v1-20260720" gpus = 1 gpu_model = "H20" hosts = ["dash4"] -command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true" +command = "TRACES_ROOT=/home/admin/cpfs/wjh/aituner/workload-regime-20260720/traces-per-gpu-low SIM_GENERATION=per-gpu-v1 PREDICTOR_CACHE_GENERATION=v4 bash runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh 4 true" artifacts = [] diff --git a/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh index 357741f..56e7460 100644 --- a/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh +++ b/runs/frontier-workload-regime-taxonomy-v0/run_simulator_group.sh @@ -25,6 +25,7 @@ ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piece CUSTOM_PYTHON_DEPS="${CUSTOM_PYTHON_DEPS:-${EXPERIMENT_ROOT}/python-deps-v2}" BASE_PYTHON_DEPS="${BASE_PYTHON_DEPS:-/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2/lib/python3.12/site-packages}" SIM_GENERATION="${SIM_GENERATION:-v4}" +PREDICTOR_CACHE_GENERATION="${PREDICTOR_CACHE_GENERATION:-${SIM_GENERATION}}" RUNNER="${CHECKOUT}/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py" for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \ @@ -35,10 +36,15 @@ for path in "${TRACES_ROOT}/manifest.json" "${FRONTIER_SOURCE}" \ done run_mode() { - local prefix_mode="$1" output_root prefix_flag family + local prefix_mode="$1" output_root predictor_cache_root prefix_flag family local -a families traces configs command output_root="${EXPERIMENT_ROOT}/sim-${SIM_GENERATION}/tp${TP}-prefix-${prefix_mode}" + predictor_cache_root="${EXPERIMENT_ROOT}/sim-${PREDICTOR_CACHE_GENERATION}/tp${TP}-prefix-${prefix_mode}/cache" mkdir -p "${output_root}/provenance" + [[ -d "${predictor_cache_root}" ]] || { + echo "ERROR: missing predictor cache ${predictor_cache_root}" >&2 + exit 1 + } if [[ "${prefix_mode}" == "true" ]]; then prefix_flag="--prefix-caching" @@ -99,6 +105,7 @@ run_mode() { --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ --python-deps "${CUSTOM_PYTHON_DEPS}" \ --output-root "${output_root}" \ + --predictor-cache-root "${predictor_cache_root}" \ "${command[@]}" "${configs[@]}" \ --rate-contract trace-window "${prefix_flag}" \ --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \