Add Qwen235 same-stack Frontier profiling campaign

runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Assemble immutable flat profiles for the Qwen235 Frontier runner."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+MODEL = "Qwen3-235B-A22B"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8"):
+        parser.add_argument(f"--{name.replace('_', '-')}", type=Path, required=True)
+    parser.add_argument("--output-root", type=Path, required=True)
+    return parser.parse_args()
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    value.update(path.read_bytes())
+    return value.hexdigest()
+
+
+def model_file(root: Path, name: str) -> Path:
+    path = root / "compute" / "h20" / MODEL / name
+    if not path.is_file():
+        raise FileNotFoundError(path)
+    return path
+
+
+def merge_csv(inputs: list[Path], output: Path) -> None:
+    fields = None
+    rows = []
+    for path in inputs:
+        with path.open(newline="") as source:
+            reader = csv.DictReader(source)
+            if fields is None:
+                fields = reader.fieldnames
+            elif reader.fieldnames != fields:
+                raise ValueError(f"CSV schema mismatch: {path}")
+            rows.extend(reader)
+    if not fields or not rows:
+        raise ValueError("cannot merge empty profile CSV")
+    with output.open("w", newline="") as target:
+        writer = csv.DictWriter(target, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def main() -> None:
+    args = parse_args()
+    output = args.output_root.resolve()
+    if output.exists():
+        raise FileExistsError(output)
+    output.mkdir(parents=True)
+    sources = {
+        "linear_op.csv": model_file(args.cuda_common, "linear_op.csv"),
+        "attention.csv": model_file(args.cuda_common, "attention.csv"),
+        "linear_op_kernel_only.csv": model_file(args.kernel_common, "linear_op_kernel_only.csv"),
+        "attention_kernel_only.csv": model_file(args.kernel_common, "attention_kernel_only.csv"),
+    }
+    for name, source in sources.items():
+        shutil.copy2(source, output / name)
+    merge_csv(
+        [model_file(args.cuda_moe_tp4, "moe.csv"), model_file(args.cuda_moe_ep8, "moe.csv")],
+        output / "moe.csv",
+    )
+    merge_csv(
+        [model_file(args.kernel_moe_tp4, "moe_kernel_only.csv"), model_file(args.kernel_moe_ep8, "moe_kernel_only.csv")],
+        output / "moe_kernel_only.csv",
+    )
+    outputs = {path.name: digest(path) for path in sorted(output.glob("*.csv"))}
+    manifest = {
+        "schema": "qwen235-v020-frontier-profile-v1",
+        "model": MODEL,
+        "measurement_families": ["CUDA_EVENT", "KERNEL_ONLY"],
+        "moe_runtime_paths": {"tp4_ep1": "TRITON", "tp8_ep8": "FLASHINFER_CUTLASS"},
+        "inputs": {name: str(getattr(args, name).resolve()) for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8")},
+        "outputs": outputs,
+    }
+    (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(manifest, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}"
+FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}"
+VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
+RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+MODEL=Qwen3-235B-A22B
+TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 256 512 1024 2048 4096 8192)
+BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128)
+KV=(128 1024 2048 4096 8192 16384 32768 40960)
+
+mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance"
+exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1
+
+echo "Q235_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} attention_tp={4,8} moe_paths={TP4/EP1:Triton,TP1/EP8:FlashInfer-CUTLASS} token_points=${#TOKENS[@]} batch_points=${#BATCHES[@]} kv_points=${#KV[@]} parallel_gpus=6 expected_wall=20-75m expected_cost=2-7_H20-GPUh output=${OUTPUT_ROOT}"
+date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ
+nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \
+  | awk '$2 > 16 {exit 1}'
+git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit"
+"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version"
+
+common_profile() {
+  local gpu="$1" method="$2" root="$3"
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \
+      --num_tokens_list "${TOKENS[@]}" --profile_method "${method}" --yes
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 8192 \
+      --batch_size_list "${BATCHES[@]}" --decode_kv_cache_size_list "${KV[@]}" \
+      --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \
+      --profile_method "${method}" --yes
+}
+
+moe_profile() {
+  local gpu="$1" method="$2" tp="$3" ep="$4" root="$5"
+  env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \
+    "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \
+      --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \
+      --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \
+      --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \
+      --num_samples_per_distribution 1 --profile_method "${method}" --yes
+}
+
+declare -a pids=()
+common_profile 0 cuda_event "${OUTPUT_ROOT}/cuda-common" > "${OUTPUT_ROOT}/logs/cuda-common.log" 2>&1 & pids+=("$!")
+common_profile 1 record_function "${OUTPUT_ROOT}/kernel-common" > "${OUTPUT_ROOT}/logs/kernel-common.log" 2>&1 & pids+=("$!")
+moe_profile 2 cuda_event 4 1 "${OUTPUT_ROOT}/cuda-moe-tp4" > "${OUTPUT_ROOT}/logs/cuda-moe-tp4.log" 2>&1 & pids+=("$!")
+moe_profile 3 cuda_event 1 8 "${OUTPUT_ROOT}/cuda-moe-ep8" > "${OUTPUT_ROOT}/logs/cuda-moe-ep8.log" 2>&1 & pids+=("$!")
+moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_ROOT}/logs/kernel-moe-tp4.log" 2>&1 & pids+=("$!")
+moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!")
+failed=0
+for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done
+[[ "${failed}" -eq 0 ]] || { tail -80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; }
+
+"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \
+  --cuda-common "${OUTPUT_ROOT}/cuda-common" \
+  --cuda-moe-tp4 "${OUTPUT_ROOT}/cuda-moe-tp4" --cuda-moe-ep8 "${OUTPUT_ROOT}/cuda-moe-ep8" \
+  --kernel-common "${OUTPUT_ROOT}/kernel-common" \
+  --kernel-moe-tp4 "${OUTPUT_ROOT}/kernel-moe-tp4" --kernel-moe-ep8 "${OUTPUT_ROOT}/kernel-moe-ep8" \
+  --output-root "${OUTPUT_ROOT}/frozen"
+date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ
+echo Q235_V020_PROFILES_COMPLETE