From 81f3a5c76d07aeea25cfb5ae30bab642dcbbcf28 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 11:45:08 +0800 Subject: [PATCH] Profile Qwen235 true mixed attention --- ...ugment_qwen235_v020_true_mixed_profiles.py | 100 ++++++++++++++++++ .../run_frontier_qwen235_v020_surface.py | 6 +- .../run_qwen235_v020_true_mixed_profiles.sh | 51 +++++++++ 3 files changed, 155 insertions(+), 2 deletions(-) create mode 100755 runs/frontier-fidelity-envelope-v1/augment_qwen235_v020_true_mixed_profiles.py create mode 100755 runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh diff --git a/runs/frontier-fidelity-envelope-v1/augment_qwen235_v020_true_mixed_profiles.py b/runs/frontier-fidelity-envelope-v1/augment_qwen235_v020_true_mixed_profiles.py new file mode 100755 index 0000000..2182737 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/augment_qwen235_v020_true_mixed_profiles.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Append measured true-mixed attention rows to an immutable Qwen235 profile.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import shutil +from pathlib import Path + + +MODEL = "Qwen3-235B-A22B" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-frozen", type=Path, required=True) + parser.add_argument("--cuda-tp4", type=Path, required=True) + parser.add_argument("--cuda-tp8", type=Path, required=True) + parser.add_argument("--kernel-tp4", type=Path, required=True) + parser.add_argument("--kernel-tp8", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + return parser.parse_args() + + +def digest(path: Path) -> str: + value = hashlib.sha256() + value.update(path.read_bytes()) + return value.hexdigest() + + +def read_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]: + with path.open(newline="") as source: + reader = csv.DictReader(source) + return list(reader.fieldnames or []), list(reader) + + +def merge_csv(inputs: list[Path], output: Path, measurement_type: str) -> int: + fields: list[str] = [] + rows: list[dict[str, str]] = [] + true_mixed_count = 0 + for index, path in enumerate(inputs): + input_fields, input_rows = read_rows(path) + for field in input_fields: + if field not in fields: + fields.append(field) + if index: + for row in input_rows: + if row.get("is_true_mixed_batch", "").lower() != "true": + raise ValueError(f"non-true-mixed row in {path}") + if row.get("measurement_type") != measurement_type: + raise ValueError(f"measurement type mismatch in {path}") + true_mixed_count += len(input_rows) + rows.extend(input_rows) + if not fields or not rows or not true_mixed_count: + raise ValueError("cannot assemble an empty true-mixed augmentation") + with output.open("w", newline="") as target: + writer = csv.DictWriter(target, fieldnames=fields, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return true_mixed_count + + +def main() -> None: + args = parse_args() + output = args.output_root.resolve() + if output.exists(): + raise FileExistsError(output) + output.mkdir(parents=True) + for name in ("linear_op.csv", "linear_op_kernel_only.csv", "moe.csv", "moe_kernel_only.csv"): + shutil.copy2(args.base_frozen / name, output / name) + cuda_count = merge_csv( + [args.base_frozen / "attention.csv", args.cuda_tp4, args.cuda_tp8], + output / "attention.csv", + "CUDA_EVENT", + ) + kernel_count = merge_csv( + [args.base_frozen / "attention_kernel_only.csv", args.kernel_tp4, args.kernel_tp8], + output / "attention_kernel_only.csv", + "KERNEL_ONLY", + ) + base_manifest = json.loads((args.base_frozen / "manifest.json").read_text()) + manifest = { + **base_manifest, + "schema": "qwen235-v020-frontier-profile-v2-true-mixed", + "true_mixed_inputs": { + name: str(getattr(args, name).resolve()) + for name in ("cuda_tp4", "cuda_tp8", "kernel_tp4", "kernel_tp8") + }, + "true_mixed_rows": {"CUDA_EVENT": cuda_count, "KERNEL_ONLY": kernel_count}, + "outputs": {path.name: digest(path) for path in sorted(output.glob("*.csv"))}, + } + (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py index 3ec8eb5..e275421 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py @@ -181,8 +181,10 @@ def main() -> None: run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" if args.resume and result_path.is_file(): - results.append(json.loads(result_path.read_text())) - continue + previous = json.loads(result_path.read_text()) + if previous.get("status") == "completed": + results.append(previous) + continue run_dir.mkdir(parents=True, exist_ok=True) command = builder.build_frontier_command( python_bin="/usr/bin/python3", diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh new file mode 100755 index 0000000..6f7e47c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_true_mixed_profiles.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL=Qwen3-235B-A22B +PREFILL_BATCHES=(1 2 4 8) +PREFILL_CHUNKS=(128 512 2048 8192) +DECODE_BATCHES=(1 8 32 64 120) +DECODE_KV=(128 1024 4096 16384 32768) + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" +exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1 + +echo "Q235_TRUE_MIXED_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} tp={4,8} grid=4x4x5x5 parallel_gpus=4 expected_wall=10-30m expected_cost=0.7-2_H20-GPUh output=${OUTPUT_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \ + | awk '$2 > 16 {exit 1}' +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version" +cd "${FRONTIER_SOURCE}" + +profile() { + local gpu="$1" method="$2" tp="$3" root="$4" + env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \ + --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ + --num_tensor_parallel_workers "${tp}" --max_model_len 40960 --max_seq_len 40960 \ + --max_batch_size 128 --batch_size_list 1 --decode_kv_cache_size_list 128 \ + --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \ + --enable_true_mixed \ + --true_mixed_prefill_batch_sizes "${PREFILL_BATCHES[@]}" \ + --true_mixed_prefill_chunk_sizes "${PREFILL_CHUNKS[@]}" \ + --true_mixed_decode_batch_sizes "${DECODE_BATCHES[@]}" \ + --true_mixed_decode_kv_cache_sizes "${DECODE_KV[@]}" \ + --true_mixed_prefill_kv_cache_size 0 \ + --profile_method "${method}" --yes +} + +declare -a pids=() +profile 0 cuda_event 4 "${OUTPUT_ROOT}/cuda-tp4" > "${OUTPUT_ROOT}/logs/cuda-tp4.log" 2>&1 & pids+=("$!") +profile 1 record_function 4 "${OUTPUT_ROOT}/kernel-tp4" > "${OUTPUT_ROOT}/logs/kernel-tp4.log" 2>&1 & pids+=("$!") +profile 2 cuda_event 8 "${OUTPUT_ROOT}/cuda-tp8" > "${OUTPUT_ROOT}/logs/cuda-tp8.log" 2>&1 & pids+=("$!") +profile 3 record_function 8 "${OUTPUT_ROOT}/kernel-tp8" > "${OUTPUT_ROOT}/logs/kernel-tp8.log" 2>&1 & pids+=("$!") +failed=0 +for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done +[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; } +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_V020_TRUE_MIXED_PROFILES_COMPLETE