Set Qwen235 profile KV cache layout

runs/frontier-fidelity-envelope-v1/augment_qwen235_v020_true_mixed_profiles.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Append measured true-mixed attention rows to an immutable Qwen235 profile."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+MODEL = "Qwen3-235B-A22B"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-frozen", type=Path, required=True)
+    parser.add_argument("--cuda-tp4", type=Path, required=True)
+    parser.add_argument("--cuda-tp8", type=Path, required=True)
+    parser.add_argument("--kernel-tp4", type=Path, required=True)
+    parser.add_argument("--kernel-tp8", type=Path, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    return parser.parse_args()
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    value.update(path.read_bytes())
+    return value.hexdigest()
+
+
+def read_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]:
+    with path.open(newline="") as source:
+        reader = csv.DictReader(source)
+        return list(reader.fieldnames or []), list(reader)
+
+
+def merge_csv(inputs: list[Path], output: Path, measurement_type: str) -> int:
+    fields: list[str] = []
+    rows: list[dict[str, str]] = []
+    true_mixed_count = 0
+    for index, path in enumerate(inputs):
+        input_fields, input_rows = read_rows(path)
+        for field in input_fields:
+            if field not in fields:
+                fields.append(field)
+        if index:
+            for row in input_rows:
+                if row.get("is_true_mixed_batch", "").lower() != "true":
+                    raise ValueError(f"non-true-mixed row in {path}")
+                if row.get("measurement_type") != measurement_type:
+                    raise ValueError(f"measurement type mismatch in {path}")
+            true_mixed_count += len(input_rows)
+        rows.extend(input_rows)
+    if not fields or not rows or not true_mixed_count:
+        raise ValueError("cannot assemble an empty true-mixed augmentation")
+    with output.open("w", newline="") as target:
+        writer = csv.DictWriter(target, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+    return true_mixed_count
+
+
+def main() -> None:
+    args = parse_args()
+    output = args.output_root.resolve()
+    if output.exists():
+        raise FileExistsError(output)
+    output.mkdir(parents=True)
+    for name in ("linear_op.csv", "linear_op_kernel_only.csv", "moe.csv", "moe_kernel_only.csv"):
+        shutil.copy2(args.base_frozen / name, output / name)
+    cuda_count = merge_csv(
+        [args.base_frozen / "attention.csv", args.cuda_tp4, args.cuda_tp8],
+        output / "attention.csv",
+        "CUDA_EVENT",
+    )
+    kernel_count = merge_csv(
+        [args.base_frozen / "attention_kernel_only.csv", args.kernel_tp4, args.kernel_tp8],
+        output / "attention_kernel_only.csv",
+        "KERNEL_ONLY",
+    )
+    base_manifest = json.loads((args.base_frozen / "manifest.json").read_text())
+    manifest = {
+        **base_manifest,
+        "schema": "qwen235-v020-frontier-profile-v2-true-mixed",
+        "true_mixed_inputs": {
+            name: str(getattr(args, name).resolve())
+            for name in ("cuda_tp4", "cuda_tp8", "kernel_tp4", "kernel_tp8")
+        },
+        "true_mixed_rows": {"CUDA_EVENT": cuda_count, "KERNEL_ONLY": kernel_count},
+        "outputs": {path.name: digest(path) for path in sorted(output.glob("*.csv"))},
+    }
+    (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(manifest, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py

@@ -171,7 +171,7 @@ async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[d
     import aiohttp
 
     timeout = aiohttp.ClientTimeout(total=args.timeout_seconds)
-    connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300)
+    connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300, force_close=True)
     benchmark_start = asyncio.get_running_loop().time() + 2.0
     async with aiohttp.ClientSession(
         base_url=f"http://{args.host}:{args.port}",
@@ -229,6 +229,7 @@ def main() -> None:
             "arrival": "original_trace_timestamp_and_order",
             "input_output_prompt": "exact_source_values",
             "served_model_alias": args.served_model,
+            "http_connection_reuse": False,
             "ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
             "tpot_slo_ms": args.tpot_slo_ms,
             "target_pass_rate": TARGET_PASS_RATE,

runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py

@@ -112,6 +112,7 @@ def knobs(config: Config, paths: dict[str, Path], contract: dict, cache: Path, p
         "batch_size_cap": config.mns,
         "max_tokens_in_batch": 8192,
         "long_prefill_token_threshold": 0,
+        "enable_chunked_prefill": True,
         "block_size": 16,
         "num_blocks_mode": "explicit",
         "num_blocks": int(resolved["num_gpu_blocks"]),
@@ -180,8 +181,10 @@ def main() -> None:
             run_dir = args.output_root / "runs" / config.name / trace["label"]
             result_path = run_dir / "result.json"
             if args.resume and result_path.is_file():
-                results.append(json.loads(result_path.read_text()))
-                continue
+                previous = json.loads(result_path.read_text())
+                if previous.get("status") == "completed":
+                    results.append(previous)
+                    continue
             run_dir.mkdir(parents=True, exist_ok=True)
             command = builder.build_frontier_command(
                 python_bin="/usr/bin/python3",

runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh

@@ -10,6 +10,13 @@ RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
 FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
 VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 
+# FlashInfer's TensorRT-LLM MoE runtime invokes `nvcc` by name when it
+# autotunes a previously unseen FP8 grouped-GEMM shape.
+export PATH="/usr/local/cuda/bin:${PATH}"
+export FRONTIER_VLLM_MODEL_TYPE=qwen3_moe
+export VLLM_KV_CACHE_LAYOUT=NHD
+command -v nvcc >/dev/null
+
 mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}"
 exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1
 echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}"

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh

@@ -53,8 +53,8 @@ run_real() {
 }
 run_real fixed-pd false 9300
 run_real fixed-po false 9400
-run_real trace-pd true 9500
-run_real trace-po true 9600
+run_real trace-pd true 9800
+run_real trace-po true 9900
 
 "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \
   --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh

@@ -30,12 +30,14 @@ common_profile() {
     "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
       --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \
-      --num_tokens_list "${TOKENS[@]}" --profile_method "${method}" --yes
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" \
+      --profile_method "${method}" --yes
   env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
     "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
-      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 8192 \
-      --batch_size_list "${BATCHES[@]}" --decode_kv_cache_size_list "${KV[@]}" \
+      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 40960 \
+      --max_batch_size 256 --batch_size_list "${BATCHES[@]}" \
+      --decode_kv_cache_size_list "${KV[@]}" \
       --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
       --profile_method "${method}" --yes
 }
@@ -46,7 +48,7 @@ moe_profile() {
     "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \
       --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
       --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \
-      --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
+      --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
       --num_samples_per_distribution 1 --profile_method "${method}" --yes
 }
 
@@ -59,7 +61,7 @@ moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_RO
 moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!")
 failed=0
 for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
-[[ "${failed}" -eq 0 ]] || { tail -80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
 
 "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \
   --cuda-common "${OUTPUT_ROOT}/cuda-common" \

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh

@@ -15,10 +15,14 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}"
 SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}"
 IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}"
+IDLE_GPU_SETTLE_ATTEMPTS="${IDLE_GPU_SETTLE_ATTEMPTS:-120}"
 RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}"
 PORT="${BASE_PORT:-9300}"
 REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")"
 
+export PATH="/usr/local/cuda/bin:${PATH}"
+command -v nvcc >/dev/null
+
 [[ "$(wc -l < "${TRACE_ROOT}/tp8/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || {
   echo 'ERROR: TP-specific request counts differ' >&2
   exit 1
@@ -40,10 +44,18 @@ PY
 }
 
 preflight_gpus() {
+  local attempt
   nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \
     | tee -a "${OUT}/controller.log"
-  nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
-    | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'
+  for ((attempt = 1; attempt <= IDLE_GPU_SETTLE_ATTEMPTS; attempt++)); do
+    if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \
+      | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then
+      return 0
+    fi
+    sleep 1
+  done
+  echo "ERROR: GPU memory did not settle below ${IDLE_GPU_MEMORY_TOLERANCE_MIB} MiB after ${IDLE_GPU_SETTLE_ATTEMPTS}s" >&2
+  return 1
 }
 
 assert_no_server() {
@@ -60,7 +72,8 @@ wait_for_wave() {
 launch_config() {
   local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1
   if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi
-  local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}"
+  local config="tp${tp}_ep${ep_size}_mns${mns}"
+  local run_out="${OUT}/real/${config}/trial${trial}"
   local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}"
   PORT=$((PORT + 1))
   if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then
@@ -97,12 +110,16 @@ run_trial() {
     "${CASE_NAME}" "${trial}" "${mnss[0]}" "${mnss[1]}" | tee -a "${OUT}/controller.log"
   launch_config "${trial}" 4 "${mnss[0]}" '0,1,2,3'
   launch_config "${trial}" 4 "${mnss[1]}" '4,5,6,7'
-  wait_for_wave && preflight_gpus && assert_no_server
+  wait_for_wave
+  preflight_gpus
+  assert_no_server
   for mns in "${mnss[@]}"; do
     printf 'WAVE_START case=%s trial=%s tp=8 mns=%s\n' \
       "${CASE_NAME}" "${trial}" "${mns}" | tee -a "${OUT}/controller.log"
     launch_config "${trial}" 8 "${mns}" '0,1,2,3,4,5,6,7'
-    wait_for_wave && preflight_gpus && assert_no_server
+    wait_for_wave
+    preflight_gpus
+    assert_no_server
   done
 }
 

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+MODEL=Qwen3-235B-A22B
+PREFILL_BATCHES=(1 2 4 8)
+PREFILL_CHUNKS=(128 512 2048 8192)
+DECODE_BATCHES=(1 8 32 64 120)
+DECODE_KV=(128 1024 4096 16384 32768)
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1
+
+echo "Q235_TRUE_MIXED_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} tp={4,8} grid=4x4x5x5 parallel_gpus=4 expected_wall=10-30m expected_cost=0.7-2_H20-GPUh output=${OUTPUT_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \
+  | awk '$2 > 16 {exit 1}'
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
+"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version"
+cd "${FRONTIER_SOURCE}"
+
+profile() {
+  local gpu="$1" method="$2" tp="$3" root="$4"
+  env CUDA_VISIBLE_DEVICES="${gpu}" VLLM_KV_CACHE_LAYOUT=NHD PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers "${tp}" --max_model_len 40960 --max_seq_len 40960 \
+      --max_batch_size 128 --batch_size_list 1 --decode_kv_cache_size_list 128 \
+      --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
+      --enable_true_mixed \
+      --true_mixed_prefill_batch_sizes "${PREFILL_BATCHES[@]}" \
+      --true_mixed_prefill_chunk_sizes "${PREFILL_CHUNKS[@]}" \
+      --true_mixed_decode_batch_sizes "${DECODE_BATCHES[@]}" \
+      --true_mixed_decode_kv_cache_sizes "${DECODE_KV[@]}" \
+      --true_mixed_prefill_kv_cache_size 0 \
+      --profile_method "${method}" --yes
+}
+
+declare -a pids=()
+profile 0 cuda_event 4 "${OUTPUT_ROOT}/cuda-tp4" > "${OUTPUT_ROOT}/logs/cuda-tp4.log" 2>&1 & pids+=("$!")
+profile 1 record_function 4 "${OUTPUT_ROOT}/kernel-tp4" > "${OUTPUT_ROOT}/logs/kernel-tp4.log" 2>&1 & pids+=("$!")
+profile 2 cuda_event 8 "${OUTPUT_ROOT}/cuda-tp8" > "${OUTPUT_ROOT}/logs/cuda-tp8.log" 2>&1 & pids+=("$!")
+profile 3 record_function 8 "${OUTPUT_ROOT}/kernel-tp8" > "${OUTPUT_ROOT}/logs/kernel-tp8.log" 2>&1 & pids+=("$!")
+failed=0
+for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
+[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_TRUE_MIXED_PROFILES_COMPLETE