Evaluate vLLM 0.20 profiles against Frontier
This commit is contained in:
60
docs/assets/simulator-fidelity/data.json
Normal file
60
docs/assets/simulator-fidelity/data.json
Normal file
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"schema": "simulator-fidelity-figure-data-v1",
|
||||
"objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu",
|
||||
"qwen30_mixed": {
|
||||
"sources": {
|
||||
"real": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z",
|
||||
"comparison": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json"
|
||||
},
|
||||
"configs": [
|
||||
{"name": "tp1_mns8", "tp": 1, "mns": 8, "real": 2.1, "frontier_profile_only": 1.1, "frontier_calibrated": 1.7166666666666666},
|
||||
{"name": "tp1_mns16", "tp": 1, "mns": 16, "real": 2.35, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
|
||||
{"name": "tp1_mns32", "tp": 1, "mns": 32, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
|
||||
{"name": "tp1_mns64", "tp": 1, "mns": 64, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
|
||||
{"name": "tp2_mns8", "tp": 2, "mns": 8, "real": 2.275, "frontier_profile_only": 0.0, "frontier_calibrated": 1.7416666666666667},
|
||||
{"name": "tp2_mns16", "tp": 2, "mns": 16, "real": 2.275, "frontier_profile_only": 1.1916666666666667, "frontier_calibrated": 2.3},
|
||||
{"name": "tp2_mns32", "tp": 2, "mns": 32, "real": 3.283333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
|
||||
{"name": "tp2_mns64", "tp": 2, "mns": 64, "real": 3.2583333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
|
||||
{"name": "tp4_mns8", "tp": 4, "mns": 8, "real": 1.2833333333333334, "frontier_profile_only": 0.0, "frontier_calibrated": 1.3208333333333333},
|
||||
{"name": "tp4_mns16", "tp": 4, "mns": 16, "real": 2.441666666666667, "frontier_profile_only": 0.0, "frontier_calibrated": 2.5},
|
||||
{"name": "tp4_mns32", "tp": 4, "mns": 32, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5},
|
||||
{"name": "tp4_mns64", "tp": 4, "mns": 64, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5}
|
||||
],
|
||||
"profile_only_metrics": {
|
||||
"kendall_tau_b": 0.0,
|
||||
"pairwise_exact_sign_accuracy": 0.3787878787878788,
|
||||
"simulator_top_set": ["tp4_mns32", "tp4_mns64"],
|
||||
"real_top_set": ["tp2_mns32"],
|
||||
"top1_regret_worst": 0.25634517766497456
|
||||
},
|
||||
"calibrated_metrics": {
|
||||
"kendall_tau_b": 0.9668009539030813,
|
||||
"pairwise_exact_sign_accuracy": 0.9393939393939394,
|
||||
"simulator_top_set": ["tp2_mns32", "tp2_mns64"],
|
||||
"real_top_set": ["tp2_mns32"],
|
||||
"top1_regret_best": 0.0,
|
||||
"top1_regret_worst": 0.0076142131979695165
|
||||
}
|
||||
},
|
||||
"qwen235_prefill": {
|
||||
"source": "runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json",
|
||||
"configs": [
|
||||
{"name": "tp4_mns64_mbt8192", "tp": 4, "mns": 64, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
|
||||
{"name": "tp4_mns128_mbt8192", "tp": 4, "mns": 128, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
|
||||
{"name": "tp4_mns64_mbt16384", "tp": 4, "mns": 64, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
|
||||
{"name": "tp4_mns128_mbt16384", "tp": 4, "mns": 128, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
|
||||
{"name": "tp8_mns64_mbt8192", "tp": 8, "mns": 64, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
|
||||
{"name": "tp8_mns128_mbt8192", "tp": 8, "mns": 128, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
|
||||
{"name": "tp8_mns64_mbt16384", "tp": 8, "mns": 64, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625},
|
||||
{"name": "tp8_mns128_mbt16384", "tp": 8, "mns": 128, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625}
|
||||
],
|
||||
"metrics": {
|
||||
"spearman_rank_correlation": 0.9486832980505138,
|
||||
"pairwise_non_tied_accuracy": 1.0,
|
||||
"comparable_non_tied_pairs": 20,
|
||||
"simulator_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
|
||||
"real_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
|
||||
"top1_regret_worst": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 152 KiB |
BIN
docs/assets/simulator-fidelity/qwen30-mixed-config-ranking.png
Normal file
BIN
docs/assets/simulator-fidelity/qwen30-mixed-config-ranking.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 183 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run and score the 12 frozen Frontier P1 primary probes, CPU only."""
|
||||
"""Run and score a frozen Frontier probe manifest, CPU only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -54,9 +54,16 @@ def git_capture(root: Path, *arguments: str) -> str:
|
||||
|
||||
|
||||
def execute(args: argparse.Namespace) -> dict[str, Any]:
|
||||
args.output = args.output.resolve()
|
||||
prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8"))
|
||||
if prepared["status"] != "PASS":
|
||||
raise RuntimeError("prepared simulator manifest did not pass")
|
||||
expected_runs = int(prepared.get("expected_runs", len(prepared["entries"])))
|
||||
if expected_runs != len(prepared["entries"]):
|
||||
raise RuntimeError(
|
||||
f"prepared manifest expected {expected_runs} runs but contains "
|
||||
f"{len(prepared['entries'])} entries"
|
||||
)
|
||||
driver = load_module(
|
||||
"simfid_execution_driver",
|
||||
args.replayserve_root
|
||||
@@ -71,7 +78,7 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
|
||||
failures = []
|
||||
gpu_visibility_disabled = True
|
||||
for sequence, entry in enumerate(prepared["entries"]):
|
||||
run_root = args.output / f"{sequence:02d}_{entry['fixture_id']}"
|
||||
run_root = args.output / f"{sequence:03d}_{entry['fixture_id']}"
|
||||
scorer_path = run_root / "scorer_output.json"
|
||||
if scorer_path.is_file() and args.resume:
|
||||
scorer = json.loads(scorer_path.read_text(encoding="utf-8"))
|
||||
@@ -205,8 +212,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
|
||||
red_flags = []
|
||||
if failures:
|
||||
red_flags.append("frontier_run_failure")
|
||||
if len(results) != 12:
|
||||
red_flags.append("runs_not_12")
|
||||
if len(results) != expected_runs:
|
||||
red_flags.append("runs_not_expected")
|
||||
if any(not 0.0 <= value <= 1.0 for value in pass_rates):
|
||||
red_flags.append("pass_rate_out_of_range")
|
||||
if any(value <= 0 for value in throughputs):
|
||||
@@ -251,7 +258,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"distinct_n": len(set(runtimes)),
|
||||
},
|
||||
"invariants": {
|
||||
"runs_12": len(results) == 12,
|
||||
"runs_expected": len(results) == expected_runs,
|
||||
"expected_runs": expected_runs,
|
||||
"zero_failures": not failures,
|
||||
"ratios_bounded": all(0.0 <= value <= 1.0 for value in pass_rates),
|
||||
"nonnegative_metrics": all(value > 0 for value in throughputs),
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare Frontier operator predictions at the identical initial scheduler state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PATTERN = re.compile(
|
||||
r"\[OP-TRACE\]\[MONOLITHIC\]\[(?:ATTENTION|MOE)\]\[([^]]+)\] "
|
||||
r"batch_id=(\d+), layer_id=(\d+), predicted_time_ms=([0-9.eE+-]+)"
|
||||
)
|
||||
COMPONENTS = (
|
||||
"input_layernorm",
|
||||
"attn_pre_proj",
|
||||
"attn_rope",
|
||||
"attn_kv_cache_save",
|
||||
"attn_prefill",
|
||||
"attn_decode",
|
||||
"attn_post_proj",
|
||||
"post_attention_layernorm",
|
||||
"moe_gating_linear",
|
||||
"moe_gating_routing_topk",
|
||||
"moe_shuffling",
|
||||
"moe_grouped_gemm",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--old", type=Path, required=True)
|
||||
parser.add_argument("--new", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def first_state(path: Path) -> dict[str, float]:
|
||||
values: dict[str, set[float]] = {}
|
||||
with path.open(errors="replace") as handle:
|
||||
for line in handle:
|
||||
match = PATTERN.search(line)
|
||||
if match and int(match[2]) == 0 and int(match[3]) == 0:
|
||||
values.setdefault(match[1], set()).add(float(match[4]))
|
||||
ambiguous = {name: sorted(items) for name, items in values.items() if len(items) != 1}
|
||||
if ambiguous:
|
||||
raise ValueError(f"ambiguous initial predictions in {path}: {ambiguous}")
|
||||
result = {name: next(iter(items)) for name, items in values.items()}
|
||||
result.setdefault("attn_decode", 0.0)
|
||||
missing = sorted(set(COMPONENTS) - set(result))
|
||||
if missing:
|
||||
raise ValueError(f"missing initial predictions in {path}: {missing}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
old = first_state(args.old)
|
||||
new = first_state(args.new)
|
||||
rows = []
|
||||
for component in COMPONENTS:
|
||||
rows.append(
|
||||
{
|
||||
"component": component,
|
||||
"historical_profile_ms": old[component],
|
||||
"vllm020_profile_ms": new[component],
|
||||
"new_over_old": new[component] / old[component]
|
||||
if old[component] != 0
|
||||
else None,
|
||||
}
|
||||
)
|
||||
old_total = sum(old[name] for name in COMPONENTS)
|
||||
new_total = sum(new[name] for name in COMPONENTS)
|
||||
output = {
|
||||
"schema": "frontier-initial-op-trace-delta.v1",
|
||||
"comparison_contract": {
|
||||
"fixture": "fidelity_p1_tp1_mns64_low1",
|
||||
"scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge",
|
||||
"calibration_a_tp": 1.0,
|
||||
},
|
||||
"layer_component_sum_ms": {
|
||||
"historical_profile": old_total,
|
||||
"vllm020_profile": new_total,
|
||||
"new_over_old": new_total / old_total,
|
||||
},
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyze real, calibrated, old-profile, and new-profile P1 probe outcomes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MODES = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--controller-state", type=Path, required=True)
|
||||
parser.add_argument("--calibrated", type=Path, required=True)
|
||||
parser.add_argument("--old-profile-only", type=Path, required=True)
|
||||
parser.add_argument("--new-profile-only", type=Path, required=True)
|
||||
parser.add_argument("--output-json", type=Path, required=True)
|
||||
parser.add_argument("--output-csv", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_results(path: Path) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
payload = json.loads(path.read_text())
|
||||
if payload["status"] != "PASS" or len(payload["results"]) != 12:
|
||||
raise ValueError(f"expected 12 passing simulator probes in {path}")
|
||||
return {(row["cell"], row["role"]): row for row in payload["results"]}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
controller = json.loads(args.controller_state.read_text())
|
||||
real: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for cell, value in controller["cells"].items():
|
||||
tp = int(value["tp"])
|
||||
for run in value["runs"]:
|
||||
if run["role"] in ("low1", "high1"):
|
||||
real[(cell, run["role"])] = {
|
||||
**run,
|
||||
"tp": tp,
|
||||
"offered_req_s_per_gpu": int(run["selected_count"]) / 60 / tp,
|
||||
}
|
||||
if len(real) != 12:
|
||||
raise ValueError(f"expected 12 real P1 probes, found {len(real)}")
|
||||
|
||||
modes = {
|
||||
"historical-calibrated": load_results(args.calibrated),
|
||||
"historical-profile-only": load_results(args.old_profile_only),
|
||||
"vllm020-profile-only": load_results(args.new_profile_only),
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
summaries: dict[str, Any] = {}
|
||||
for mode in MODES:
|
||||
predicted = modes[mode]
|
||||
false_feasible = 0
|
||||
false_infeasible = 0
|
||||
pass_errors: list[float] = []
|
||||
capacity_lower_bounds: dict[str, float] = {}
|
||||
for key in sorted(real):
|
||||
real_row = real[key]
|
||||
sim_row = predicted[key]
|
||||
scorer = sim_row["scorer"]
|
||||
sim_feasible = bool(scorer["slo"]["feasible"])
|
||||
real_feasible = bool(real_row["feasible"])
|
||||
false_feasible += int(sim_feasible and not real_feasible)
|
||||
false_infeasible += int(real_feasible and not sim_feasible)
|
||||
pass_error = abs(float(scorer["slo"]["pass_rate"]) - float(real_row["pass_rate"]))
|
||||
pass_errors.append(pass_error)
|
||||
rows.append(
|
||||
{
|
||||
"mode": mode,
|
||||
"cell": key[0],
|
||||
"role": key[1],
|
||||
"real_feasible": real_feasible,
|
||||
"sim_feasible": sim_feasible,
|
||||
"real_pass_rate": float(real_row["pass_rate"]),
|
||||
"sim_pass_rate": float(scorer["slo"]["pass_rate"]),
|
||||
"pass_rate_absolute_error": pass_error,
|
||||
"offered_req_s_per_gpu": float(real_row["offered_req_s_per_gpu"]),
|
||||
"sim_throughput_req_s_per_gpu": float(
|
||||
scorer["throughput_requests_per_second_per_gpu"]
|
||||
),
|
||||
}
|
||||
)
|
||||
if sim_feasible:
|
||||
capacity_lower_bounds[key[0]] = max(
|
||||
capacity_lower_bounds.get(key[0], 0.0),
|
||||
float(real_row["offered_req_s_per_gpu"]),
|
||||
)
|
||||
agreement = 12 - false_feasible - false_infeasible
|
||||
summaries[mode] = {
|
||||
"probe_classification": {
|
||||
"agreement": agreement,
|
||||
"accuracy": agreement / 12,
|
||||
"false_feasible": false_feasible,
|
||||
"false_infeasible": false_infeasible,
|
||||
},
|
||||
"pass_rate_mae": statistics.mean(pass_errors),
|
||||
"feasible_probe_count": sum(
|
||||
bool(row["scorer"]["slo"]["feasible"]) for row in predicted.values()
|
||||
),
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
cell: capacity_lower_bounds.get(cell, 0.0)
|
||||
for cell in sorted(controller["cells"])
|
||||
},
|
||||
"rank_identifiable": bool(capacity_lower_bounds),
|
||||
}
|
||||
|
||||
output = {
|
||||
"schema": "frontier-qwen30-p1-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": 6,
|
||||
"probes_per_cell": 2,
|
||||
"roles": ["low1", "high1"],
|
||||
"reading": "held-out boundary classification, not a complete capacity sweep",
|
||||
},
|
||||
"sources": {
|
||||
"controller_state": str(args.controller_state.resolve()),
|
||||
"historical_calibrated": str(args.calibrated.resolve()),
|
||||
"historical_profile_only": str(args.old_profile_only.resolve()),
|
||||
"vllm020_profile_only": str(args.new_profile_only.resolve()),
|
||||
},
|
||||
"summaries": summaries,
|
||||
"rows": rows,
|
||||
}
|
||||
args.output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_json.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
args.output_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output_csv.open("w", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(args.output_json)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare captured vLLM expert loads with Frontier's fixed routing prior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--routing", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def gini(values: np.ndarray) -> float:
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
if values.sum() == 0:
|
||||
return 0.0
|
||||
ordered = np.sort(values)
|
||||
n = len(ordered)
|
||||
indices = np.arange(1, n + 1, dtype=np.float64)
|
||||
return float((2 * np.sum(indices * ordered) / np.sum(ordered) - (n + 1)) / n)
|
||||
|
||||
|
||||
def stats(values: np.ndarray) -> dict[str, float]:
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
mean = float(np.mean(values))
|
||||
return {
|
||||
"load_cv": float(np.std(values) / mean),
|
||||
"load_gini": gini(values),
|
||||
"max_load_ratio": float(np.max(values) / mean),
|
||||
"expert_utilization": float(np.count_nonzero(values) / len(values)),
|
||||
}
|
||||
|
||||
|
||||
def proportional_counts(total: int, ratios: np.ndarray) -> np.ndarray:
|
||||
exact = total * ratios / ratios.sum()
|
||||
counts = np.floor(exact).astype(np.int64)
|
||||
remainder = total - int(counts.sum())
|
||||
order = sorted(
|
||||
range(len(ratios)),
|
||||
key=lambda i: (-(exact[i] - counts[i]), -(ratios[i] / ratios.sum()), i),
|
||||
)
|
||||
for index in range(remainder):
|
||||
counts[order[index % len(order)]] += 1
|
||||
assert int(counts.sum()) == total
|
||||
return counts
|
||||
|
||||
|
||||
def correlation(left: np.ndarray, right: np.ndarray) -> float:
|
||||
value = float(np.corrcoef(left, right)[0, 1])
|
||||
return value if math.isfinite(value) else 0.0
|
||||
|
||||
|
||||
def distribution(values: list[float]) -> dict[str, float]:
|
||||
return {
|
||||
"min": min(values),
|
||||
"median": statistics.median(values),
|
||||
"max": max(values),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
routing = json.loads(args.routing.read_text())
|
||||
phases = routing["phases"]
|
||||
layer_count = len(phases["prefill"]["per_layer"])
|
||||
expert_count = len(phases["prefill"]["per_layer"][0]["counts"])
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for phase in ("prefill", "decode"):
|
||||
for layer, actual in enumerate(phases[phase]["per_layer"]):
|
||||
rng = np.random.RandomState(args.seed + layer)
|
||||
ratios = rng.uniform(0.1, 1.0, expert_count)
|
||||
synthetic = proportional_counts(int(actual["total_routed_tokens"]), ratios)
|
||||
synthetic_stats = stats(synthetic)
|
||||
for name in synthetic_stats:
|
||||
if not math.isclose(
|
||||
stats(np.asarray(actual["counts"]))[name],
|
||||
float(actual[name]),
|
||||
rel_tol=0,
|
||||
abs_tol=1e-12,
|
||||
):
|
||||
raise ValueError(f"captured metric mismatch: {phase} layer {layer} {name}")
|
||||
rows.append(
|
||||
{
|
||||
"phase": phase,
|
||||
"layer": layer,
|
||||
"total_routed_tokens": int(actual["total_routed_tokens"]),
|
||||
"actual": {name: float(actual[name]) for name in synthetic_stats},
|
||||
"frontier_simulation": synthetic_stats,
|
||||
"actual_vs_frontier_pearson": correlation(
|
||||
np.asarray(actual["counts"], dtype=np.float64), synthetic
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
phase_summary: dict[str, Any] = {}
|
||||
for phase in ("prefill", "decode"):
|
||||
selected = [row for row in rows if row["phase"] == phase]
|
||||
phase_summary[phase] = {
|
||||
"token_count": int(phases[phase]["token_count"]),
|
||||
"actual": {
|
||||
name: distribution([row["actual"][name] for row in selected])
|
||||
for name in selected[0]["actual"]
|
||||
},
|
||||
"frontier_simulation": {
|
||||
name: distribution([row["frontier_simulation"][name] for row in selected])
|
||||
for name in selected[0]["frontier_simulation"]
|
||||
},
|
||||
"actual_vs_frontier_pearson": distribution(
|
||||
[row["actual_vs_frontier_pearson"] for row in selected]
|
||||
),
|
||||
}
|
||||
|
||||
phase_correlations = []
|
||||
for layer in range(layer_count):
|
||||
prefill = np.asarray(phases["prefill"]["per_layer"][layer]["counts"])
|
||||
decode = np.asarray(phases["decode"]["per_layer"][layer]["counts"])
|
||||
phase_correlations.append(correlation(prefill, decode))
|
||||
|
||||
output = {
|
||||
"schema": "frontier-routing-mismatch.v1",
|
||||
"source": str(args.routing.resolve()),
|
||||
"frontier_contract": {
|
||||
"mode": "simulation",
|
||||
"seed": args.seed,
|
||||
"allocation": "per-layer fixed Uniform(0.1, 1.0), normalized once and reused for every batch and phase",
|
||||
"layer_count": layer_count,
|
||||
"expert_count": expert_count,
|
||||
},
|
||||
"phase_summary": phase_summary,
|
||||
"actual_prefill_vs_decode_pearson": distribution(phase_correlations),
|
||||
"frontier_prefill_vs_decode_pearson": 1.0,
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Score a replacement-profile S2 sweep against the frozen real oracle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--shard-metrics", type=Path, action="append", required=True)
|
||||
parser.add_argument("--ground-truth", type=Path, required=True)
|
||||
parser.add_argument("--historical-metrics", type=Path, required=True)
|
||||
parser.add_argument("--historical-analyzer", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_module(path: Path):
|
||||
spec = importlib.util.spec_from_file_location("simfid_s2_analyzer", path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def ranking(scores: dict[str, float]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"rank": index + 1, "cell": cell, "score": score}
|
||||
for index, (cell, score) in enumerate(
|
||||
sorted(scores.items(), key=lambda item: (-item[1], item[0]))
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def robust_loao(
|
||||
analyzer: Any,
|
||||
runs: list[dict[str, Any]],
|
||||
mode: str,
|
||||
reading: str,
|
||||
real_scores: dict[str, float],
|
||||
) -> dict[str, Any]:
|
||||
"""Historical LOAO with an explicit null range for all-tied concordance."""
|
||||
anchors = sorted(
|
||||
{(row["cell_id"], int(row["probe_index"])) for row in runs},
|
||||
key=lambda item: (analyzer.CELL_ORDER.index(item[0]), item[1]),
|
||||
)
|
||||
replicates = []
|
||||
undefined = []
|
||||
for cell, probe in anchors:
|
||||
scores, detail = analyzer.cell_scores(
|
||||
runs, mode, reading, removed=(cell, probe)
|
||||
)
|
||||
if scores is None:
|
||||
undefined.append(
|
||||
{"removed_cell_id": cell, "removed_probe_index": probe, **detail}
|
||||
)
|
||||
continue
|
||||
metric = analyzer.rank_metrics(real_scores, scores)
|
||||
replicates.append(
|
||||
{
|
||||
"removed_cell_id": cell,
|
||||
"removed_probe_index": probe,
|
||||
"top1_optimistic_regret": metric["top1"]["optimistic_regret"],
|
||||
"top1_worst_case_regret": metric["top1"]["worst_case_regret"],
|
||||
"top5_minimum_exact_five_overlap": metric["top5"]["minimum_exact_five_overlap"],
|
||||
"top5_maximum_exact_five_overlap": metric["top5"]["maximum_exact_five_overlap"],
|
||||
"top5_optimistic_regret": metric["top5"]["optimistic_regret"],
|
||||
"top5_worst_case_regret": metric["top5"]["worst_case_regret"],
|
||||
"tau_b": metric["kendall_tau_b"]["tau_b"],
|
||||
"pairwise_exact_sign_accuracy": metric["pairwise_direction"]["exact_sign_accuracy"],
|
||||
"pairwise_non_tied_concordance": metric["pairwise_direction"]["non_tied_concordance"],
|
||||
"trap_reproduced": metric["named_interactions"]["trap_reproduced"],
|
||||
"tp2_mns32_unique_global_best": metric["named_interactions"]["tp2_mns32_unique_global_best"],
|
||||
}
|
||||
)
|
||||
scalar_keys = (
|
||||
"top1_optimistic_regret",
|
||||
"top1_worst_case_regret",
|
||||
"top5_minimum_exact_five_overlap",
|
||||
"top5_maximum_exact_five_overlap",
|
||||
"top5_optimistic_regret",
|
||||
"top5_worst_case_regret",
|
||||
"tau_b",
|
||||
"pairwise_exact_sign_accuracy",
|
||||
"pairwise_non_tied_concordance",
|
||||
)
|
||||
ranges = {}
|
||||
for key in scalar_keys:
|
||||
values = [row[key] for row in replicates if row[key] is not None]
|
||||
ranges[key] = {
|
||||
"min": min(values) if values else None,
|
||||
"max": max(values) if values else None,
|
||||
}
|
||||
return {
|
||||
"replicate_count": len(anchors),
|
||||
"defined_replicates": len(replicates),
|
||||
"undefined_replicates": len(undefined),
|
||||
"undefined": undefined,
|
||||
"ranges": ranges,
|
||||
"trap_reproduced_count": sum(row["trap_reproduced"] for row in replicates),
|
||||
"tp2_mns32_unique_global_best_count": sum(
|
||||
row["tp2_mns32_unique_global_best"] for row in replicates
|
||||
),
|
||||
"replicates": replicates,
|
||||
"range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
analyzer = load_module(args.historical_analyzer)
|
||||
ground = json.loads(args.ground_truth.read_text())
|
||||
cells = {str(cell["cell_id"]): cell for cell in ground["cells"]}
|
||||
if set(cells) != set(analyzer.CELL_ORDER):
|
||||
raise ValueError("ground-truth cells differ from the frozen 3x4 surface")
|
||||
|
||||
result_rows: list[dict[str, Any]] = []
|
||||
sources = []
|
||||
for path in args.shard_metrics:
|
||||
payload = json.loads(path.read_text())
|
||||
if payload["status"] != "PASS":
|
||||
raise ValueError(f"shard did not pass: {path}")
|
||||
sources.append({"path": str(path.resolve()), "runs": len(payload["results"])})
|
||||
result_rows.extend(payload["results"])
|
||||
if len(result_rows) != 92:
|
||||
raise ValueError(f"expected 92 replacement-profile probes, found {len(result_rows)}")
|
||||
|
||||
seen: set[tuple[str, int]] = set()
|
||||
runs = []
|
||||
for row in result_rows:
|
||||
cell_id = str(row["cell"])
|
||||
probe_index = int(row["probe_index"])
|
||||
key = (cell_id, probe_index)
|
||||
if key in seen:
|
||||
raise ValueError(f"duplicate probe {key}")
|
||||
seen.add(key)
|
||||
real_probe = cells[cell_id]["probe_history"][probe_index]
|
||||
if not math.isclose(
|
||||
float(real_probe["sampling_u"]), float(row["sampling_u"]), rel_tol=0, abs_tol=1e-15
|
||||
):
|
||||
raise ValueError(f"sampling_u mismatch for {key}")
|
||||
if int(real_probe["request_count"]) != int(row["selected_count"]):
|
||||
raise ValueError(f"request count mismatch for {key}")
|
||||
runs.append(
|
||||
{
|
||||
**row,
|
||||
"cell_id": cell_id,
|
||||
"mode": "vllm020-profile-only",
|
||||
"probe_index": probe_index,
|
||||
"sampling_u": float(row["sampling_u"]),
|
||||
"request_count": int(row["selected_count"]),
|
||||
"tensor_parallel_size": int(row["tensor_parallel_size"]),
|
||||
"real_anchor": {
|
||||
"feasible": bool(real_probe["feasible"]),
|
||||
"pass_rate": float(real_probe["pass_rate"]),
|
||||
"request_count": int(real_probe["request_count"]),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
historical = json.loads(args.historical_metrics.read_text())
|
||||
real_scores = {cell: float(value) for cell, value in historical["real_scores"].items()}
|
||||
analyses = {}
|
||||
for reading in analyzer.READINGS:
|
||||
scores, detail = analyzer.cell_scores(runs, "vllm020-profile-only", reading)
|
||||
if scores is None:
|
||||
raise ValueError(f"undefined {reading} scores: {detail}")
|
||||
analysis = {
|
||||
"reading": reading,
|
||||
"simulated_scores": scores,
|
||||
"ranking": ranking(scores),
|
||||
"cell_score_details": detail["cells"],
|
||||
"metrics": analyzer.rank_metrics(real_scores, scores),
|
||||
"loao": robust_loao(
|
||||
analyzer, runs, "vllm020-profile-only", reading, real_scores
|
||||
),
|
||||
}
|
||||
if reading == "SLO-gated":
|
||||
analysis["false_feasibility"] = analyzer.false_feasibility(
|
||||
runs, "vllm020-profile-only"
|
||||
)
|
||||
analyses[reading] = analysis
|
||||
|
||||
historical_modes = {}
|
||||
for label, key in (
|
||||
("historical-profile-only", "uncalibrated/SLO-gated"),
|
||||
("historical-per-tp-calibration", "frozen-calibrated/SLO-gated"),
|
||||
):
|
||||
value = historical["analyses"][key]
|
||||
historical_modes[label] = {
|
||||
"simulated_scores": value["simulated_scores"],
|
||||
"ranking": ranking(value["simulated_scores"]),
|
||||
"metrics": value["metrics"],
|
||||
"false_feasibility": value["false_feasibility"],
|
||||
}
|
||||
|
||||
output = {
|
||||
"schema": "frontier-qwen30-s2-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": len(analyzer.CELL_ORDER),
|
||||
"probes": len(runs),
|
||||
"trace_horizon_seconds": 60,
|
||||
"calibration_a_tp": 1.0,
|
||||
},
|
||||
"sources": {
|
||||
"replacement_profile_shards": sources,
|
||||
"ground_truth": str(args.ground_truth.resolve()),
|
||||
"historical_metrics": str(args.historical_metrics.resolve()),
|
||||
},
|
||||
"real_scores": real_scores,
|
||||
"historical_modes": historical_modes,
|
||||
"vllm020_profile_only": analyses,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"comparison_contract": {
|
||||
"calibration_a_tp": 1.0,
|
||||
"fixture": "fidelity_p1_tp1_mns64_low1",
|
||||
"scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge"
|
||||
},
|
||||
"layer_component_sum_ms": {
|
||||
"historical_profile": 1.148334,
|
||||
"new_over_old": 2.700194368537377,
|
||||
"vllm020_profile": 3.1007249999999997
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"component": "input_layernorm",
|
||||
"historical_profile_ms": 0.010619,
|
||||
"new_over_old": 1.955833882663151,
|
||||
"vllm020_profile_ms": 0.020769
|
||||
},
|
||||
{
|
||||
"component": "attn_pre_proj",
|
||||
"historical_profile_ms": 0.204389,
|
||||
"new_over_old": 1.0199276869107439,
|
||||
"vllm020_profile_ms": 0.208462
|
||||
},
|
||||
{
|
||||
"component": "attn_rope",
|
||||
"historical_profile_ms": 0.018908,
|
||||
"new_over_old": 7.394171779141103,
|
||||
"vllm020_profile_ms": 0.139809
|
||||
},
|
||||
{
|
||||
"component": "attn_kv_cache_save",
|
||||
"historical_profile_ms": 0.014226,
|
||||
"new_over_old": 1.427386475467454,
|
||||
"vllm020_profile_ms": 0.020306
|
||||
},
|
||||
{
|
||||
"component": "attn_prefill",
|
||||
"historical_profile_ms": 0.107759,
|
||||
"new_over_old": 2.670709639102071,
|
||||
"vllm020_profile_ms": 0.287793
|
||||
},
|
||||
{
|
||||
"component": "attn_decode",
|
||||
"historical_profile_ms": 0.0,
|
||||
"new_over_old": null,
|
||||
"vllm020_profile_ms": 0.0
|
||||
},
|
||||
{
|
||||
"component": "attn_post_proj",
|
||||
"historical_profile_ms": 0.134153,
|
||||
"new_over_old": 0.967559428413826,
|
||||
"vllm020_profile_ms": 0.129801
|
||||
},
|
||||
{
|
||||
"component": "post_attention_layernorm",
|
||||
"historical_profile_ms": 0.011481,
|
||||
"new_over_old": 0.9000087100426792,
|
||||
"vllm020_profile_ms": 0.010333
|
||||
},
|
||||
{
|
||||
"component": "moe_gating_linear",
|
||||
"historical_profile_ms": 0.011802,
|
||||
"new_over_old": 2.2642772411455687,
|
||||
"vllm020_profile_ms": 0.026723
|
||||
},
|
||||
{
|
||||
"component": "moe_gating_routing_topk",
|
||||
"historical_profile_ms": 0.009662,
|
||||
"new_over_old": 2.2922790312564687,
|
||||
"vllm020_profile_ms": 0.022148
|
||||
},
|
||||
{
|
||||
"component": "moe_shuffling",
|
||||
"historical_profile_ms": 0.030431,
|
||||
"new_over_old": 0.0,
|
||||
"vllm020_profile_ms": 0.0
|
||||
},
|
||||
{
|
||||
"component": "moe_grouped_gemm",
|
||||
"historical_profile_ms": 0.594904,
|
||||
"new_over_old": 3.756204362384519,
|
||||
"vllm020_profile_ms": 2.234581
|
||||
}
|
||||
],
|
||||
"schema": "frontier-initial-op-trace-delta.v1"
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
mode,cell,role,real_feasible,sim_feasible,real_pass_rate,sim_pass_rate,pass_rate_absolute_error,offered_req_s_per_gpu,sim_throughput_req_s_per_gpu
|
||||
historical-calibrated,tp1_mns64,high1,False,True,0.24581005586592178,0.9888268156424581,0.7430167597765364,2.9833333333333334,2.8823345959582403
|
||||
historical-calibrated,tp1_mns64,low1,True,True,1.0,1.0,0.0,2.033333333333333,1.9806287931485522
|
||||
historical-calibrated,tp1_mns8,high1,False,False,0.20670391061452514,0.16759776536312848,0.03910614525139666,2.9833333333333334,2.162661799726284
|
||||
historical-calibrated,tp1_mns8,low1,True,True,1.0,0.9754098360655737,0.024590163934426257,2.033333333333333,1.9806313066623409
|
||||
historical-calibrated,tp2_mns64,high1,True,True,1.0,1.0,0.0,2.875,2.809583898291986
|
||||
historical-calibrated,tp2_mns64,low1,True,True,1.0,1.0,0.0,1.9583333333333333,1.911740196570019
|
||||
historical-calibrated,tp2_mns8,high1,False,False,0.20238095238095238,0.15476190476190477,0.047619047619047616,2.8,2.083217332903509
|
||||
historical-calibrated,tp2_mns8,low1,True,True,1.0,0.9868995633187773,0.013100436681222738,1.9083333333333334,1.8637394408880168
|
||||
historical-calibrated,tp4_mns16,high1,False,False,0.10666666666666667,0.2693333333333333,0.16266666666666663,3.125,2.6829294600763345
|
||||
historical-calibrated,tp4_mns16,low1,False,True,0.6196078431372549,1.0,0.3803921568627451,2.125,2.0865667413849516
|
||||
historical-calibrated,tp4_mns64,high1,True,True,1.0,1.0,0.0,3.125,3.0718260341342285
|
||||
historical-calibrated,tp4_mns64,low1,True,True,1.0,1.0,0.0,2.125,2.0967062495193307
|
||||
historical-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.24581005586592178,0.0,2.9833333333333334,2.82546810606982
|
||||
historical-profile-only,tp1_mns64,low1,True,False,1.0,0.7377049180327869,0.2622950819672131,2.033333333333333,1.9427723213024963
|
||||
historical-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.09497206703910614,0.111731843575419,2.9833333333333334,1.5728783582441022
|
||||
historical-profile-only,tp1_mns8,low1,True,False,1.0,0.1721311475409836,0.8278688524590164,2.033333333333333,1.5629164685645112
|
||||
historical-profile-only,tp2_mns64,high1,True,False,1.0,0.2608695652173913,0.7391304347826086,2.875,2.7121507155597584
|
||||
historical-profile-only,tp2_mns64,low1,True,False,1.0,0.8382978723404255,0.16170212765957448,1.9583333333333333,1.849101894293262
|
||||
historical-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.041666666666666664,0.16071428571428573,2.8,0.9859710483293953
|
||||
historical-profile-only,tp2_mns8,low1,True,False,1.0,0.0611353711790393,0.9388646288209607,1.9083333333333334,0.9771135206177947
|
||||
historical-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.028,0.07866666666666668,3.125,0.9604809617478992
|
||||
historical-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.058823529411764705,0.5607843137254902,2.125,0.9495221159963836
|
||||
historical-profile-only,tp4_mns64,high1,True,False,1.0,0.0026666666666666666,0.9973333333333333,3.125,2.281094127221267
|
||||
historical-profile-only,tp4_mns64,low1,True,False,1.0,0.4627450980392157,0.5372549019607843,2.125,1.9793960278798877
|
||||
vllm020-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.0,0.24581005586592178,2.9833333333333334,2.074240230678028
|
||||
vllm020-profile-only,tp1_mns64,low1,True,False,1.0,0.0,1.0,2.033333333333333,1.6189525827713862
|
||||
vllm020-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.0,0.20670391061452514,2.9833333333333334,0.5271192714936628
|
||||
vllm020-profile-only,tp1_mns8,low1,True,False,1.0,0.0,1.0,2.033333333333333,0.5225514701978402
|
||||
vllm020-profile-only,tp2_mns64,high1,True,False,1.0,0.0,1.0,2.875,1.7237810220598582
|
||||
vllm020-profile-only,tp2_mns64,low1,True,False,1.0,0.0,1.0,1.9583333333333333,1.608919184557297
|
||||
vllm020-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.0,0.20238095238095238,2.8,0.39315152956277477
|
||||
vllm020-profile-only,tp2_mns8,low1,True,False,1.0,0.0,1.0,1.9083333333333334,0.3894201655965407
|
||||
vllm020-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.0,0.10666666666666667,3.125,0.43700447014112276
|
||||
vllm020-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.0,0.6196078431372549,2.125,0.43475717185302865
|
||||
vllm020-profile-only,tp4_mns64,high1,True,False,1.0,0.0,1.0,3.125,1.2598449015844073
|
||||
vllm020-profile-only,tp4_mns64,low1,True,False,1.0,0.0,1.0,2.125,1.2025831680037373
|
||||
|
@@ -0,0 +1,511 @@
|
||||
{
|
||||
"rows": [
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.7430167597765364,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9888268156424581,
|
||||
"sim_throughput_req_s_per_gpu": 2.8823345959582403
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.9806287931485522
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.03910614525139666,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.16759776536312848,
|
||||
"sim_throughput_req_s_per_gpu": 2.162661799726284
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.024590163934426257,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9754098360655737,
|
||||
"sim_throughput_req_s_per_gpu": 1.9806313066623409
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.809583898291986
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.911740196570019
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.047619047619047616,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.15476190476190477,
|
||||
"sim_throughput_req_s_per_gpu": 2.083217332903509
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 0.013100436681222738,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9868995633187773,
|
||||
"sim_throughput_req_s_per_gpu": 1.8637394408880168
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.16266666666666663,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.2693333333333333,
|
||||
"sim_throughput_req_s_per_gpu": 2.6829294600763345
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.3803921568627451,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.0865667413849516
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 3.0718260341342285
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.0967062495193307
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.24581005586592178,
|
||||
"sim_throughput_req_s_per_gpu": 2.82546810606982
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.2622950819672131,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.7377049180327869,
|
||||
"sim_throughput_req_s_per_gpu": 1.9427723213024963
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.111731843575419,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.09497206703910614,
|
||||
"sim_throughput_req_s_per_gpu": 1.5728783582441022
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.8278688524590164,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.1721311475409836,
|
||||
"sim_throughput_req_s_per_gpu": 1.5629164685645112
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 0.7391304347826086,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.2608695652173913,
|
||||
"sim_throughput_req_s_per_gpu": 2.7121507155597584
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 0.16170212765957448,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.8382978723404255,
|
||||
"sim_throughput_req_s_per_gpu": 1.849101894293262
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.16071428571428573,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.041666666666666664,
|
||||
"sim_throughput_req_s_per_gpu": 0.9859710483293953
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 0.9388646288209607,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0611353711790393,
|
||||
"sim_throughput_req_s_per_gpu": 0.9771135206177947
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.07866666666666668,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.028,
|
||||
"sim_throughput_req_s_per_gpu": 0.9604809617478992
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.5607843137254902,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.058823529411764705,
|
||||
"sim_throughput_req_s_per_gpu": 0.9495221159963836
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.9973333333333333,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0026666666666666666,
|
||||
"sim_throughput_req_s_per_gpu": 2.281094127221267
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.5372549019607843,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.4627450980392157,
|
||||
"sim_throughput_req_s_per_gpu": 1.9793960278798877
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.24581005586592178,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.074240230678028
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.6189525827713862
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.20670391061452514,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.5271192714936628
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.5225514701978402
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.7237810220598582
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.608919184557297
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.20238095238095238,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.39315152956277477
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.3894201655965407
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.10666666666666667,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.43700447014112276
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.6196078431372549,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.43475717185302865
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.2598449015844073
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.2025831680037373
|
||||
}
|
||||
],
|
||||
"schema": "frontier-qwen30-p1-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": 6,
|
||||
"probes_per_cell": 2,
|
||||
"reading": "held-out boundary classification, not a complete capacity sweep",
|
||||
"roles": [
|
||||
"low1",
|
||||
"high1"
|
||||
]
|
||||
},
|
||||
"sources": {
|
||||
"controller_state": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/controller-state.json",
|
||||
"historical_calibrated": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/results/metrics.json",
|
||||
"historical_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/old-profile-only-v2/results/metrics.json",
|
||||
"vllm020_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2/results/metrics.json"
|
||||
},
|
||||
"summaries": {
|
||||
"historical-calibrated": {
|
||||
"feasible_probe_count": 9,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 2.9833333333333334,
|
||||
"tp1_mns8": 2.033333333333333,
|
||||
"tp2_mns64": 2.875,
|
||||
"tp2_mns8": 1.9083333333333334,
|
||||
"tp4_mns16": 2.125,
|
||||
"tp4_mns64": 3.125
|
||||
},
|
||||
"pass_rate_mae": 0.11754094806600345,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.8333333333333334,
|
||||
"agreement": 10,
|
||||
"false_feasible": 2,
|
||||
"false_infeasible": 0
|
||||
},
|
||||
"rank_identifiable": true
|
||||
},
|
||||
"historical-profile-only": {
|
||||
"feasible_probe_count": 0,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 0.0,
|
||||
"tp1_mns8": 0.0,
|
||||
"tp2_mns64": 0.0,
|
||||
"tp2_mns8": 0.0,
|
||||
"tp4_mns16": 0.0,
|
||||
"tp4_mns64": 0.0
|
||||
},
|
||||
"pass_rate_mae": 0.44802887255544604,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.4166666666666667,
|
||||
"agreement": 5,
|
||||
"false_feasible": 0,
|
||||
"false_infeasible": 7
|
||||
},
|
||||
"rank_identifiable": false
|
||||
},
|
||||
"vllm020-profile-only": {
|
||||
"feasible_probe_count": 0,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 0.0,
|
||||
"tp1_mns8": 0.0,
|
||||
"tp2_mns64": 0.0,
|
||||
"tp2_mns8": 0.0,
|
||||
"tp4_mns16": 0.0,
|
||||
"tp4_mns64": 0.0
|
||||
},
|
||||
"pass_rate_mae": 0.69843078572211,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.4166666666666667,
|
||||
"agreement": 5,
|
||||
"false_feasible": 0,
|
||||
"false_infeasible": 7
|
||||
},
|
||||
"rank_identifiable": false
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,469 @@
|
||||
{
|
||||
"block_size": 64,
|
||||
"cohort_output": "/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/trace-support/cohort.jsonl",
|
||||
"cohort_sha256": "9ef15a4c8843127a5c52ff8455bdeb9945ac6921f2977dd11070ed6944eed7c8",
|
||||
"full_downsampled_cohort": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 15.0,
|
||||
"p50": 38.0,
|
||||
"p75": 82.0,
|
||||
"p90": 111.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8186.0,
|
||||
"min": 71.0,
|
||||
"p10": 208.0,
|
||||
"p25": 933.0,
|
||||
"p50": 2421.0,
|
||||
"p75": 5244.0,
|
||||
"p90": 7103.0,
|
||||
"p95": 7775.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 0.20259999999998257,
|
||||
"min": 0.06279999999997088,
|
||||
"p10": 0.0917999999999708,
|
||||
"p25": 0.10240000000003846,
|
||||
"p50": 0.1163999999999703,
|
||||
"p75": 0.13009999999999167,
|
||||
"p90": 0.14210000000002765,
|
||||
"p95": 0.15119999999997447,
|
||||
"p99": 0.17959999999997933
|
||||
},
|
||||
"multi_turn_fraction": 0.140625,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.09274807669777795,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 55.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 49.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3520.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 3136.0
|
||||
},
|
||||
"rows_with_hash_ids": 512,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 25607,
|
||||
"unique_blocks": 23232
|
||||
},
|
||||
"request_count": 512,
|
||||
"sampling_u": {
|
||||
"max": 0.9979913161468553,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.10365756051317851,
|
||||
"p25": 0.22877823992809823,
|
||||
"p50": 0.4667483893139273,
|
||||
"p75": 0.7338360613813898,
|
||||
"p90": 0.8969698510725516,
|
||||
"p95": 0.9558957048665733,
|
||||
"p99": 0.991130680217395
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"Trace length/hash support does not determine dynamic decode or mixed batch shapes.",
|
||||
"Those shapes depend jointly on arrival history, SLO pressure, TP execution time, MNS, chunking, and KV eviction.",
|
||||
"MoE expert routing is not present in the trace and must be measured from model execution."
|
||||
],
|
||||
"loader_contract": {
|
||||
"completion_tokens_override": 128,
|
||||
"downsampling": "AITuner _downsample_requests before sampling_u threshold",
|
||||
"input_length_filter": {
|
||||
"max": 8192,
|
||||
"min": 0
|
||||
},
|
||||
"max_requests_per_probe": 512,
|
||||
"ordering": "arrival_s",
|
||||
"replay_time_scale": 0.1
|
||||
},
|
||||
"schema_version": "qwen30_trace_profile_support.v1",
|
||||
"study": "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/configs/examples/dash0_qwen30b_a3b_community_vllm020_noharness.json",
|
||||
"study_sha256": "cd0b9e2385751478f459faeae2cf3d12c58229866176911051421bc043bf99d4",
|
||||
"threshold_cohorts": {
|
||||
"0.125": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 127.0,
|
||||
"min": 2.0,
|
||||
"p10": 7.0,
|
||||
"p25": 15.0,
|
||||
"p50": 30.0,
|
||||
"p75": 76.0,
|
||||
"p90": 105.0,
|
||||
"p95": 111.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8110.0,
|
||||
"min": 74.0,
|
||||
"p10": 392.0,
|
||||
"p25": 922.0,
|
||||
"p50": 1864.0,
|
||||
"p75": 4844.0,
|
||||
"p90": 6686.0,
|
||||
"p95": 7054.0,
|
||||
"p99": 8110.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 6.217200000000002,
|
||||
"min": 0.10639999999998517,
|
||||
"p10": 0.16680000000001272,
|
||||
"p25": 0.30810000000001736,
|
||||
"p50": 0.6001999999999938,
|
||||
"p75": 1.1201000000000016,
|
||||
"p90": 1.8216999999999643,
|
||||
"p95": 2.419699999999967,
|
||||
"p99": 6.217200000000002
|
||||
},
|
||||
"multi_turn_fraction": 0.15151515151515152,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.10466666666666667,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 13.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 13.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 832.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 832.0
|
||||
},
|
||||
"rows_with_hash_ids": 66,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 3000,
|
||||
"unique_blocks": 2686
|
||||
},
|
||||
"request_count": 66,
|
||||
"sampling_u": {
|
||||
"max": 0.12321850600323363,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.01599560440427848,
|
||||
"p25": 0.03636425746001375,
|
||||
"p50": 0.06451196858246099,
|
||||
"p75": 0.10162658487005978,
|
||||
"p90": 0.1138971596689021,
|
||||
"p95": 0.12001874701158237,
|
||||
"p99": 0.12321850600323363
|
||||
}
|
||||
},
|
||||
"0.25": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 14.0,
|
||||
"p50": 34.0,
|
||||
"p75": 87.0,
|
||||
"p90": 107.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"p10": 197.0,
|
||||
"p25": 867.0,
|
||||
"p50": 2167.0,
|
||||
"p75": 5568.0,
|
||||
"p90": 6841.0,
|
||||
"p95": 7745.0,
|
||||
"p99": 8110.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 2.3110999999999855,
|
||||
"min": 0.07340000000003499,
|
||||
"p10": 0.11439999999997497,
|
||||
"p25": 0.14210000000002765,
|
||||
"p50": 0.3411000000000044,
|
||||
"p75": 0.5650000000000093,
|
||||
"p90": 0.8215000000000146,
|
||||
"p95": 1.088799999999992,
|
||||
"p99": 1.7577999999999996
|
||||
},
|
||||
"multi_turn_fraction": 0.11888111888111888,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.07953890489913544,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 13.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 13.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 832.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 832.0
|
||||
},
|
||||
"rows_with_hash_ids": 143,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 6940,
|
||||
"unique_blocks": 6388
|
||||
},
|
||||
"request_count": 143,
|
||||
"sampling_u": {
|
||||
"max": 0.24941730181025415,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.03095529459918134,
|
||||
"p25": 0.07431293234041883,
|
||||
"p50": 0.13301201741611407,
|
||||
"p75": 0.1994148366998705,
|
||||
"p90": 0.2292686392189136,
|
||||
"p95": 0.24107176388894497,
|
||||
"p99": 0.24671643798343437
|
||||
}
|
||||
},
|
||||
"0.5": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 2.0,
|
||||
"p25": 15.0,
|
||||
"p50": 42.0,
|
||||
"p75": 86.0,
|
||||
"p90": 109.0,
|
||||
"p95": 117.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"p10": 113.0,
|
||||
"p25": 905.0,
|
||||
"p50": 2648.0,
|
||||
"p75": 5490.0,
|
||||
"p90": 6926.0,
|
||||
"p95": 7476.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 1.2561000000000142,
|
||||
"min": 0.06749999999997236,
|
||||
"p10": 0.10019999999999563,
|
||||
"p25": 0.11840000000002249,
|
||||
"p50": 0.15819999999994394,
|
||||
"p75": 0.2599999999999483,
|
||||
"p90": 0.3867999999999938,
|
||||
"p95": 0.5083999999999804,
|
||||
"p99": 0.696700000000007
|
||||
},
|
||||
"multi_turn_fraction": 0.14130434782608695,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.08318264014466546,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 52.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 24.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3328.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 1536.0
|
||||
},
|
||||
"rows_with_hash_ids": 276,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 13825,
|
||||
"unique_blocks": 12675
|
||||
},
|
||||
"request_count": 276,
|
||||
"sampling_u": {
|
||||
"max": 0.49832600818075473,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.056494245471656455,
|
||||
"p25": 0.1313132001019981,
|
||||
"p50": 0.24403265916374703,
|
||||
"p75": 0.3706937275751198,
|
||||
"p90": 0.4530058956953501,
|
||||
"p95": 0.48171254599865826,
|
||||
"p99": 0.49639910750753047
|
||||
}
|
||||
},
|
||||
"1.0": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 15.0,
|
||||
"p50": 38.0,
|
||||
"p75": 82.0,
|
||||
"p90": 111.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8186.0,
|
||||
"min": 71.0,
|
||||
"p10": 208.0,
|
||||
"p25": 933.0,
|
||||
"p50": 2421.0,
|
||||
"p75": 5244.0,
|
||||
"p90": 7103.0,
|
||||
"p95": 7775.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 0.20259999999998257,
|
||||
"min": 0.06279999999997088,
|
||||
"p10": 0.0917999999999708,
|
||||
"p25": 0.10240000000003846,
|
||||
"p50": 0.1163999999999703,
|
||||
"p75": 0.13009999999999167,
|
||||
"p90": 0.14210000000002765,
|
||||
"p95": 0.15119999999997447,
|
||||
"p99": 0.17959999999997933
|
||||
},
|
||||
"multi_turn_fraction": 0.140625,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.09274807669777795,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 55.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 49.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3520.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 3136.0
|
||||
},
|
||||
"rows_with_hash_ids": 512,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 25607,
|
||||
"unique_blocks": 23232
|
||||
},
|
||||
"request_count": 512,
|
||||
"sampling_u": {
|
||||
"max": 0.9979913161468553,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.10365756051317851,
|
||||
"p25": 0.22877823992809823,
|
||||
"p50": 0.4667483893139273,
|
||||
"p75": 0.7338360613813898,
|
||||
"p90": 0.8969698510725516,
|
||||
"p95": 0.9558957048665733,
|
||||
"p99": 0.991130680217395
|
||||
}
|
||||
}
|
||||
},
|
||||
"trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl",
|
||||
"trace_sha256": "f539f38eb0ee0f750e3c23ff47df6eed3faf723a25f1444d55665a85871750b9",
|
||||
"window_id": "chat_w20260311_1000"
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# 实验 EXP-SIMFID-Q30-P020：同栈 per-TP operator profile 能否恢复 Frontier ranking？
|
||||
|
||||
> **状态：** 运行中（用户已批准 5-step campaign）
|
||||
> **状态：** 完成；H1 rejected，结果支持 H2（2026-07-16）
|
||||
|
||||
## Claim 与决策
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
|
||||
## 预期产物与 review
|
||||
|
||||
- **预期数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；12-cell simulation outputs；三 baseline comparison table。
|
||||
- **Figure prototype：** `mock-profile-ablation.png`；x 为 12 configs，y 为 normalized capacity；series 为 real、old profile-only、new profile-only、calibrated。图中数值明确标为 mock，不进入结论。
|
||||
- **实际数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；92/92 new profile-only runs；old/new/calibrated comparison；scheduler graph-mode 与 native MoE routing diagnostics。
|
||||
- **Figure：** `../../docs/assets/simulator-fidelity/qwen30-vllm020-profile-ablation.png`；四个 panels 分别展示完整 ranking、routing skew、phase-dependent graph mode 和 held-out P1 boundary accuracy。图中只使用实测或冻结分析结果。
|
||||
- **人工 review：** 已批准。用户明确要求推进 5 个步骤，并要求 smoke 通过后直接完整运行。
|
||||
- **Review 意见：** 不能把 TP–MNS 耦合拆成互不相关问题；profile 只冻结 execution counterfactual 的证据，最终仍以完整 config ranking 判断。
|
||||
|
||||
@@ -32,12 +32,27 @@
|
||||
- **Code：** AITuner 当前 branch `codex/fidelity-prefix-pilot-20260714`；Frontier canonical commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e`；vLLM commit/tag `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1` / `0.20.0`。
|
||||
- **Environment：** `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；hardware/driver/package freeze 写入每个 profile artifact。
|
||||
- **产物路径：** remote `/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/`; local harvest `runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/`。
|
||||
- **已知 deviation：** 历史 real ground truth 来自 dash1；本次 profile 按用户要求在 dash0 生成。driver 也已从历史 570.133.20 漂移到 dash0 当前 580.95.05。结果只能归因于“profile stack alignment 在当前 dash0 上的效果”，不能声称严格复现旧 dash1 execution latency。
|
||||
- **已知 deviation：** 历史 real ground truth 与本次 profile 都来自 dash0；早期文档中的主机 provenance 标注错误。driver 从历史 570.133.20 漂移到当前 580.95.05，因此结果仍不能解释为严格的逐 kernel latency 复现，只能作为同一 H20 主机上 profile-stack alignment 的对照。
|
||||
|
||||
## 结果
|
||||
|
||||
- **观察事实：** 待完整 profile 与 simulation。
|
||||
- **异常：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 自带的 attention benchmark可调用实际 FlashAttention backend，但 mixed prefill+decode 在 FA3 中是一个 fused varlen call，而 Frontier schema将其拆成 `attn_prefill` 与 `attn_decode` 两项。
|
||||
- **Interpretation 与剩余 alternatives：** 这既是 profiler compatibility gap，也是潜在 representational gap。先用真实 kernel smoke 判断能否无损物化 Frontier profile；不能无损时同时保留 fused total 与 schema projection，避免误把 projection error 当 kernel error。
|
||||
- **Claim update：** unchanged。
|
||||
- **下一步：** 通过 vLLM 0.20 exact-kernel smoke；冻结 trace-derived shape support；完整 TP1/2/4 profiling；profile-only rerun。
|
||||
- **Step 1—同栈固定：** dash0 8×H20；Qwen3-30B-A3B BF16；community vLLM `0.20.0+cu129` / source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`。真实 serving 与 profiles 使用同一主机和 model family；历史 driver 570.133.20 到当前 580.95.05 的漂移保留为限制。
|
||||
- **Step 2—per-TP profiles：** profile-v2 冻结 attention 132 rows、直接测量 fused-mixed diagnostic 30 rows、linear 36 rows、MoE 72 rows、all-reduce 24 rows。attention/linear/MoE 通过单 H20 local-shard shape 表达 TP；all-reduce 使用真实 TP2/4 ranks。base comparison 保持 historical analytical CC backend，measured all-reduce 仅作独立诊断。
|
||||
- **Step 3—trace execution coverage：** 6 个 P1 cells 共 161,161 scheduler/model steps：pure decode 151,471（100% FULL graph），pure prefill 224（192 NONE、30 PIECEWISE、2 FULL），true mixed 9,466（8,623 NONE、842 PIECEWISE、1 FULL）。exact-trace 8-request native routing probe 表明实际 median max/mean expert load 为 prefill 6.12、decode 5.64，而 Frontier fixed prior 为 1.84；actual-vs-prior Pearson median 接近 0。该小 cohort 用于机制否证，不作为 routing population estimate。
|
||||
- **Step 4—冻结 profile-only：** profile-v1 因 true-mixed schema 缺少 `attn_decode_in_mixed` 在第一个 mixed batch fail。profile-v2 使用 same-TP pure prefill/decode ratio 做 total-conserving compatibility split，同时保留直接实测 fused total。两个 46-run CPU shards 均 PASS，92/92 runs 无 crash，总 simulator runtime 2,394.9 s（0.665 CPU-hour）。
|
||||
- **Step 5—ranking comparison：** real optimum 为 TP2/MNS32。old profile-only top set 为 TP4/MNS32/64，τ-b=0、worst regret=25.63%；new vLLM 0.20 profile-only 在所有 92 anchors 上都 SLO-infeasible，12 configs 全并列，τ-b=0、exact pair sign=7.58%、worst tie-break regret=60.91%；historical frozen per-TP calibration top set 为 TP2/MNS32/64，τ-b=0.9668、worst regret=0.76%。在 held-out P1 12 boundary labels 上，new profile-only accuracy 41.67%、pass-rate MAE 69.84%，也没有优于 old profile-only；calibration 分别为 83.33% 和 11.75%。
|
||||
- **Same-state delta：** 在 `tp1_mns64` 的 batch 0/layer 0、profile-dependent trajectories 尚未分叉时，new/old layer component sum 已为 2.70×；MoE grouped GEMM 3.76×、RoPE 7.39×、attention prefill 2.67×。误差不是可由一个 version correction 解释的一致 scale。
|
||||
- **异常与 schema gap：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 的 FA3 mixed prefill+decode 是一个 fused varlen call，而 Frontier schema要求 `attn_prefill` 和 `attn_decode_in_mixed` 两项。兼容 split 是归因，不是观测。
|
||||
- **Interpretation：** H1 被否证。同栈 isolated operator profile 不等价于真实 execution counterfactual。TP 与 MNS 共同改变 local shape、collective、queue/batch composition、phase mixture、routing 和 CUDA graph regime；static operator sum 在 scheduler 推进后产生 feedback amplification。历史 per-TP calibration 是 composition 之后的外部 E2E scale，能吸收这些残差，但不是 Frontier 原生 per-TP profiling 或机制解释。
|
||||
- **Claim update：** 支持 H2。新的 research target 是验证 execution-context-conditioned composition，而不是继续把更多 isolated kernel rows 当作 fidelity 的充分条件。
|
||||
- **下一步：** 分别做 graph-conditioned step profile、fused mixed total、trace-conditioned routing 和 measured collective 四个单变量消融，再回到完整 TP×MNS joint surface 评测；frozen calibration 仅作上界，不参与 mechanism fitting。
|
||||
|
||||
## 产物索引
|
||||
|
||||
- Frozen profile：`frozen/profile-v2/manifest.json`
|
||||
- Full-surface comparison：`artifacts/qwen30-s2-profile-ablation.json`
|
||||
- P1 boundary comparison：`artifacts/qwen30-p1-profile-ablation.json`
|
||||
- Scheduler/graph-mode summary：`artifacts/qwen30-p1b-opprof-summary.json`
|
||||
- Native routing mismatch：`artifacts/qwen30-routing-mismatch.json`
|
||||
- Same-state operator delta：`artifacts/qwen30-initial-op-trace-delta.json`
|
||||
- Figure script：`plot_profile_ablation.py`
|
||||
|
||||
@@ -0,0 +1,824 @@
|
||||
{
|
||||
"environment": [
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
}
|
||||
],
|
||||
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
|
||||
"rows": [
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08288000151515007,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.4872959852218628,
|
||||
"mean": 0.1310015980154276,
|
||||
"median": 0.07679999992251396,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.12790721677293843
|
||||
},
|
||||
{
|
||||
"max": 0.4402880072593689,
|
||||
"mean": 0.12842560112476348,
|
||||
"median": 0.08288000151515007,
|
||||
"min": 0.0655680000782013,
|
||||
"std": 0.11220176524616535
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0793600007891655,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12716799974441528,
|
||||
"mean": 0.07871360033750534,
|
||||
"median": 0.0759200006723404,
|
||||
"min": 0.06032000109553337,
|
||||
"std": 0.019331314939874535
|
||||
},
|
||||
{
|
||||
"max": 0.12380799651145935,
|
||||
"mean": 0.08059840016067028,
|
||||
"median": 0.0793600007891655,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.01687088356254092
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0713919997215271,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12697599828243256,
|
||||
"mean": 0.0767391998320818,
|
||||
"median": 0.07078400254249573,
|
||||
"min": 0.05910399928689003,
|
||||
"std": 0.018775178979463278
|
||||
},
|
||||
{
|
||||
"max": 0.11430399864912033,
|
||||
"mean": 0.07594559974968433,
|
||||
"median": 0.0713919997215271,
|
||||
"min": 0.06124800071120262,
|
||||
"std": 0.015719922150631036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08056000247597694,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1037760004401207,
|
||||
"mean": 0.07954559996724128,
|
||||
"median": 0.08056000247597694,
|
||||
"min": 0.05955199897289276,
|
||||
"std": 0.0135697420393132
|
||||
},
|
||||
{
|
||||
"max": 0.10608000308275223,
|
||||
"mean": 0.07971520014107228,
|
||||
"median": 0.07593599706888199,
|
||||
"min": 0.06028800085186958,
|
||||
"std": 0.015319849772775456
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0865279994904995,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14470399916172028,
|
||||
"mean": 0.09121599942445754,
|
||||
"median": 0.0865279994904995,
|
||||
"min": 0.06441599875688553,
|
||||
"std": 0.024893837894277456
|
||||
},
|
||||
{
|
||||
"max": 0.12438400089740753,
|
||||
"mean": 0.08531199917197227,
|
||||
"median": 0.08031999692320824,
|
||||
"min": 0.06364800035953522,
|
||||
"std": 0.01878029830059533
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07135999947786331,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11753600090742111,
|
||||
"mean": 0.07606079950928687,
|
||||
"median": 0.07135999947786331,
|
||||
"min": 0.05843200162053108,
|
||||
"std": 0.01755519771639284
|
||||
},
|
||||
{
|
||||
"max": 0.1103999987244606,
|
||||
"mean": 0.07607359997928143,
|
||||
"median": 0.07073600217700005,
|
||||
"min": 0.05721599981188774,
|
||||
"std": 0.016904445220949783
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07321599870920181,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11740799993276596,
|
||||
"mean": 0.07749119997024537,
|
||||
"median": 0.07203200086951256,
|
||||
"min": 0.05862399935722351,
|
||||
"std": 0.017701381594822835
|
||||
},
|
||||
{
|
||||
"max": 0.11382400244474411,
|
||||
"mean": 0.07733759954571724,
|
||||
"median": 0.07321599870920181,
|
||||
"min": 0.059039998799562454,
|
||||
"std": 0.017188890882557036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09025600180029869,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13980799913406372,
|
||||
"mean": 0.0950367994606495,
|
||||
"median": 0.09025600180029869,
|
||||
"min": 0.06815999746322632,
|
||||
"std": 0.022829835127539378
|
||||
},
|
||||
{
|
||||
"max": 0.14764800667762756,
|
||||
"mean": 0.09710080176591873,
|
||||
"median": 0.08720000088214874,
|
||||
"min": 0.07152000069618225,
|
||||
"std": 0.02500574954063789
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08083200082182884,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.15574400126934052,
|
||||
"mean": 0.08760640025138855,
|
||||
"median": 0.07846399769186974,
|
||||
"min": 0.07097599655389786,
|
||||
"std": 0.024487311423551025
|
||||
},
|
||||
{
|
||||
"max": 0.16284799575805664,
|
||||
"mean": 0.08963519930839539,
|
||||
"median": 0.08083200082182884,
|
||||
"min": 0.07100799679756165,
|
||||
"std": 0.026047320562445356
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.10891199856996536,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1582079976797104,
|
||||
"mean": 0.11541439890861512,
|
||||
"median": 0.10836799815297127,
|
||||
"min": 0.09062399715185165,
|
||||
"std": 0.018285136316576037
|
||||
},
|
||||
{
|
||||
"max": 0.1578879952430725,
|
||||
"mean": 0.11537599861621857,
|
||||
"median": 0.10891199856996536,
|
||||
"min": 0.0960640013217926,
|
||||
"std": 0.018246538626977286
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1703840047121048,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.19327999651432037,
|
||||
"mean": 0.1707327976822853,
|
||||
"median": 0.1703840047121048,
|
||||
"min": 0.14815999567508698,
|
||||
"std": 0.014211056022719618
|
||||
},
|
||||
{
|
||||
"max": 0.19276799261569977,
|
||||
"mean": 0.1658592015504837,
|
||||
"median": 0.16379200667142868,
|
||||
"min": 0.14560000598430634,
|
||||
"std": 0.013840249648693638
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.25539200007915497,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.2807359993457794,
|
||||
"mean": 0.25750079900026324,
|
||||
"median": 0.25539200007915497,
|
||||
"min": 0.24624000489711761,
|
||||
"std": 0.008925204570802302
|
||||
},
|
||||
{
|
||||
"max": 0.2863999903202057,
|
||||
"mean": 0.2585055992007256,
|
||||
"median": 0.255280002951622,
|
||||
"min": 0.24371199309825897,
|
||||
"std": 0.012059738582656496
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1021759994328022,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.9770879745483398,
|
||||
"mean": 0.1974783968180418,
|
||||
"median": 0.10099200159311295,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.2660581738745569
|
||||
},
|
||||
{
|
||||
"max": 0.892799973487854,
|
||||
"mean": 0.18164799660444259,
|
||||
"median": 0.1021759994328022,
|
||||
"min": 0.06435199826955795,
|
||||
"std": 0.23947520188197013
|
||||
},
|
||||
{
|
||||
"max": 0.6467199921607971,
|
||||
"mean": 0.15839359983801843,
|
||||
"median": 0.10100800171494484,
|
||||
"min": 0.06800000369548798,
|
||||
"std": 0.16617013141866102
|
||||
},
|
||||
{
|
||||
"max": 0.6725760102272034,
|
||||
"mean": 0.15686400160193442,
|
||||
"median": 0.10044800117611885,
|
||||
"min": 0.06063999980688095,
|
||||
"std": 0.17523222161300497
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12694399803876877,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.6659520268440247,
|
||||
"mean": 0.1927648030221462,
|
||||
"median": 0.12694399803876877,
|
||||
"min": 0.07609599828720093,
|
||||
"std": 0.16697784531907736
|
||||
},
|
||||
{
|
||||
"max": 0.695360004901886,
|
||||
"mean": 0.19356480240821838,
|
||||
"median": 0.11726400256156921,
|
||||
"min": 0.0796160027384758,
|
||||
"std": 0.17588096678867862
|
||||
},
|
||||
{
|
||||
"max": 0.5939840078353882,
|
||||
"mean": 0.1868800014257431,
|
||||
"median": 0.12379200011491776,
|
||||
"min": 0.07427199929952621,
|
||||
"std": 0.1463231714943902
|
||||
},
|
||||
{
|
||||
"max": 0.6635839939117432,
|
||||
"mean": 0.1874335989356041,
|
||||
"median": 0.12014400213956833,
|
||||
"min": 0.07526399940252304,
|
||||
"std": 0.16680959677760304
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09161599725484848,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12665599584579468,
|
||||
"mean": 0.09443839862942696,
|
||||
"median": 0.09124799817800522,
|
||||
"min": 0.06431999802589417,
|
||||
"std": 0.0205690155775906
|
||||
},
|
||||
{
|
||||
"max": 0.1303039938211441,
|
||||
"mean": 0.09712959825992584,
|
||||
"median": 0.09161599725484848,
|
||||
"min": 0.07648000121116638,
|
||||
"std": 0.019072048129173236
|
||||
},
|
||||
{
|
||||
"max": 0.13836799561977386,
|
||||
"mean": 0.09821119979023933,
|
||||
"median": 0.09148800000548363,
|
||||
"min": 0.0727040022611618,
|
||||
"std": 0.021314066545189116
|
||||
},
|
||||
{
|
||||
"max": 0.12992000579833984,
|
||||
"mean": 0.09272959977388381,
|
||||
"median": 0.08931199833750725,
|
||||
"min": 0.06406400352716446,
|
||||
"std": 0.021360803830354765
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08580800145864487,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13449600338935852,
|
||||
"mean": 0.08960640132427215,
|
||||
"median": 0.08299200236797333,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.020633597898445998
|
||||
},
|
||||
{
|
||||
"max": 0.14735999703407288,
|
||||
"mean": 0.09248319901525974,
|
||||
"median": 0.08580800145864487,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.025960234928829564
|
||||
},
|
||||
{
|
||||
"max": 0.13705599308013916,
|
||||
"mean": 0.08947199806571007,
|
||||
"median": 0.08460799977183342,
|
||||
"min": 0.0634239986538887,
|
||||
"std": 0.02122838946992339
|
||||
},
|
||||
{
|
||||
"max": 0.13846400380134583,
|
||||
"mean": 0.0859104000031948,
|
||||
"median": 0.08308799937367439,
|
||||
"min": 0.05974400043487549,
|
||||
"std": 0.022373631424433445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09867199882864952,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1279039978981018,
|
||||
"mean": 0.09748799875378608,
|
||||
"median": 0.09678399935364723,
|
||||
"min": 0.0655359998345375,
|
||||
"std": 0.02315719144614622
|
||||
},
|
||||
{
|
||||
"max": 0.1356479972600937,
|
||||
"mean": 0.10018239840865135,
|
||||
"median": 0.09532799944281578,
|
||||
"min": 0.06185600161552429,
|
||||
"std": 0.02330700253650042
|
||||
},
|
||||
{
|
||||
"max": 0.13142399489879608,
|
||||
"mean": 0.09778879955410957,
|
||||
"median": 0.09492799639701843,
|
||||
"min": 0.06560000032186508,
|
||||
"std": 0.02307579212430045
|
||||
},
|
||||
{
|
||||
"max": 0.1276479959487915,
|
||||
"mean": 0.09611519873142242,
|
||||
"median": 0.09867199882864952,
|
||||
"min": 0.0642239972949028,
|
||||
"std": 0.02273612181973431
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09646400064229965,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14060799777507782,
|
||||
"mean": 0.09770880043506622,
|
||||
"median": 0.09115200117230415,
|
||||
"min": 0.06950400024652481,
|
||||
"std": 0.024752645089849798
|
||||
},
|
||||
{
|
||||
"max": 0.14377599954605103,
|
||||
"mean": 0.09824960008263588,
|
||||
"median": 0.08999999985098839,
|
||||
"min": 0.07103999704122543,
|
||||
"std": 0.025403407389046027
|
||||
},
|
||||
{
|
||||
"max": 0.13680000603199005,
|
||||
"mean": 0.09993600100278854,
|
||||
"median": 0.09646400064229965,
|
||||
"min": 0.06790400296449661,
|
||||
"std": 0.022297424273985882
|
||||
},
|
||||
{
|
||||
"max": 0.1391039937734604,
|
||||
"mean": 0.09769919961690902,
|
||||
"median": 0.09601600095629692,
|
||||
"min": 0.06835199892520905,
|
||||
"std": 0.023899922100804445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08377600088715553,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1464959979057312,
|
||||
"mean": 0.09276160076260567,
|
||||
"median": 0.08377600088715553,
|
||||
"min": 0.06777600198984146,
|
||||
"std": 0.024751086465295658
|
||||
},
|
||||
{
|
||||
"max": 0.14319999516010284,
|
||||
"mean": 0.09080640003085136,
|
||||
"median": 0.080400001257658,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.023365718881708488
|
||||
},
|
||||
{
|
||||
"max": 0.1382399946451187,
|
||||
"mean": 0.09063360020518303,
|
||||
"median": 0.08193599805235863,
|
||||
"min": 0.06627199798822403,
|
||||
"std": 0.02372618650854302
|
||||
},
|
||||
{
|
||||
"max": 0.14313599467277527,
|
||||
"mean": 0.09044799953699112,
|
||||
"median": 0.08128000050783157,
|
||||
"min": 0.06652799993753433,
|
||||
"std": 0.02462486862033686
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1128000020980835,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1600320041179657,
|
||||
"mean": 0.11426239982247352,
|
||||
"median": 0.1128000020980835,
|
||||
"min": 0.07657600194215775,
|
||||
"std": 0.030240087702350687
|
||||
},
|
||||
{
|
||||
"max": 0.15881599485874176,
|
||||
"mean": 0.11206399947404862,
|
||||
"median": 0.10628800094127655,
|
||||
"min": 0.0772159993648529,
|
||||
"std": 0.029933135345483627
|
||||
},
|
||||
{
|
||||
"max": 0.15612800419330597,
|
||||
"mean": 0.10761600062251091,
|
||||
"median": 0.09860799834132195,
|
||||
"min": 0.07689599692821503,
|
||||
"std": 0.02634237020678647
|
||||
},
|
||||
{
|
||||
"max": 0.15865600109100342,
|
||||
"mean": 0.11094079986214637,
|
||||
"median": 0.10979199782013893,
|
||||
"min": 0.07583999633789062,
|
||||
"std": 0.028619217028542445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08755199983716011,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11999999731779099,
|
||||
"mean": 0.0843871995806694,
|
||||
"median": 0.08755199983716011,
|
||||
"min": 0.06332799792289734,
|
||||
"std": 0.016638543890716024
|
||||
},
|
||||
{
|
||||
"max": 0.1218239963054657,
|
||||
"mean": 0.08518079966306687,
|
||||
"median": 0.08032000064849854,
|
||||
"min": 0.06393600255250931,
|
||||
"std": 0.019900899429956945
|
||||
},
|
||||
{
|
||||
"max": 0.11849600076675415,
|
||||
"mean": 0.0843968003988266,
|
||||
"median": 0.08702399954199791,
|
||||
"min": 0.06297600269317627,
|
||||
"std": 0.017031908773433545
|
||||
},
|
||||
{
|
||||
"max": 0.12300799787044525,
|
||||
"mean": 0.0846304003149271,
|
||||
"median": 0.08139199763536453,
|
||||
"min": 0.06195199862122536,
|
||||
"std": 0.020106523698622265
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12361599877476692,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.17155200242996216,
|
||||
"mean": 0.12168959975242614,
|
||||
"median": 0.12055999785661697,
|
||||
"min": 0.09609600156545639,
|
||||
"std": 0.022565485532483928
|
||||
},
|
||||
{
|
||||
"max": 0.9246399998664856,
|
||||
"mean": 0.19978560134768486,
|
||||
"median": 0.12361599877476692,
|
||||
"min": 0.09715200215578079,
|
||||
"std": 0.24230694305662265
|
||||
},
|
||||
{
|
||||
"max": 0.9317439794540405,
|
||||
"mean": 0.20037759989500045,
|
||||
"median": 0.12327999994158745,
|
||||
"min": 0.09603200107812881,
|
||||
"std": 0.24450903278000383
|
||||
},
|
||||
{
|
||||
"max": 0.9321280121803284,
|
||||
"mean": 0.19875840097665787,
|
||||
"median": 0.12230399996042252,
|
||||
"min": 0.0950080007314682,
|
||||
"std": 0.24519252220785093
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.20030399411916733,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.3261440098285675,
|
||||
"mean": 0.20826880186796187,
|
||||
"median": 0.19974400103092194,
|
||||
"min": 0.1409280002117157,
|
||||
"std": 0.051397264855718945
|
||||
},
|
||||
{
|
||||
"max": 0.3248000144958496,
|
||||
"mean": 0.20548800230026246,
|
||||
"median": 0.1979840025305748,
|
||||
"min": 0.141184002161026,
|
||||
"std": 0.04980002399656717
|
||||
},
|
||||
{
|
||||
"max": 0.32547199726104736,
|
||||
"mean": 0.21280319690704347,
|
||||
"median": 0.20030399411916733,
|
||||
"min": 0.14127999544143677,
|
||||
"std": 0.05304243085685509
|
||||
},
|
||||
{
|
||||
"max": 0.26047998666763306,
|
||||
"mean": 0.1969312012195587,
|
||||
"median": 0.18079999834299088,
|
||||
"min": 0.14057600498199463,
|
||||
"std": 0.042174123638424224
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.2924960106611252,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.45052799582481384,
|
||||
"mean": 0.3013375997543335,
|
||||
"median": 0.28273600339889526,
|
||||
"min": 0.21334399282932281,
|
||||
"std": 0.07214223764527236
|
||||
},
|
||||
{
|
||||
"max": 0.4466240108013153,
|
||||
"mean": 0.2968191936612129,
|
||||
"median": 0.27796798944473267,
|
||||
"min": 0.21241599321365356,
|
||||
"std": 0.07400678240849741
|
||||
},
|
||||
{
|
||||
"max": 0.3830080032348633,
|
||||
"mean": 0.2946112036705017,
|
||||
"median": 0.2924960106611252,
|
||||
"min": 0.21084800362586975,
|
||||
"std": 0.05323627615746332
|
||||
},
|
||||
{
|
||||
"max": 0.4609600007534027,
|
||||
"mean": 0.3054272010922432,
|
||||
"median": 0.289792001247406,
|
||||
"min": 0.21062399446964264,
|
||||
"std": 0.07608482904865672
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
}
|
||||
],
|
||||
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
|
||||
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
|
@@ -0,0 +1,31 @@
|
||||
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,representation
|
||||
1,q64_8q1s512,1,8,64,8,511.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q128_8q1s1k,1,8,128,8,1023.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q128_16q1s1k,1,16,128,16,1023.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q256_16q1s2k,1,16,256,16,2047.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q256_32q1s2k,1,32,256,32,2047.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q512_32q1s4k,1,32,512,32,4095.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q512_64q1s4k,1,64,512,64,4095.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q1k_64q1s8k,1,64,1024,64,8191.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q2k_32q1s4k,1,32,2048,32,4095.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q4k_16q1s4k,1,16,4096,16,4095.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q64_8q1s512,1,8,64,8,511.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q128_8q1s1k,1,8,128,8,1023.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q128_16q1s1k,1,16,128,16,1023.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q256_16q1s2k,1,16,256,16,2047.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q256_32q1s2k,1,32,256,32,2047.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q512_32q1s4k,1,32,512,32,4095.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q512_64q1s4k,1,64,512,64,4095.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q1k_64q1s8k,1,64,1024,64,8191.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q2k_32q1s4k,1,32,2048,32,4095.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q4k_16q1s4k,1,16,4096,16,4095.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q64_8q1s512,1,8,64,8,511.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q128_8q1s1k,1,8,128,8,1023.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q128_16q1s1k,1,16,128,16,1023.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q256_16q1s2k,1,16,256,16,2047.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q256_32q1s2k,1,32,256,32,2047.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q512_32q1s4k,1,32,512,32,4095.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q512_64q1s4k,1,64,512,64,4095.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q1k_64q1s8k,1,64,1024,64,8191.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q2k_32q1s4k,1,32,2048,32,4095.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q4k_16q1s4k,1,16,4096,16,4095.0,0.456512001156807,0.456512001156807,0.017680000513792038,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
|
@@ -0,0 +1,37 @@
|
||||
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
|
||||
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
|
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "674e010598eea4c06bdee04ccba413f4c34194fbe2d70b143c6ec2169b638cc5",
|
||||
"attention_true_mixed_fused.csv": "92cd22f8b669abc7af488cf3d067ed880a6cced2dba1a3213a93a4f02a0001a8",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target",
|
||||
"attention_true_mixed": "Preserved as fused-total diagnostics and excluded from attention.csv because Frontier requires an unobservable prefill/decode split",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 102,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v1"
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
|
@@ -0,0 +1,824 @@
|
||||
{
|
||||
"environment": [
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
}
|
||||
],
|
||||
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
|
||||
"rows": [
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08288000151515007,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.4872959852218628,
|
||||
"mean": 0.1310015980154276,
|
||||
"median": 0.07679999992251396,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.12790721677293843
|
||||
},
|
||||
{
|
||||
"max": 0.4402880072593689,
|
||||
"mean": 0.12842560112476348,
|
||||
"median": 0.08288000151515007,
|
||||
"min": 0.0655680000782013,
|
||||
"std": 0.11220176524616535
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0793600007891655,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12716799974441528,
|
||||
"mean": 0.07871360033750534,
|
||||
"median": 0.0759200006723404,
|
||||
"min": 0.06032000109553337,
|
||||
"std": 0.019331314939874535
|
||||
},
|
||||
{
|
||||
"max": 0.12380799651145935,
|
||||
"mean": 0.08059840016067028,
|
||||
"median": 0.0793600007891655,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.01687088356254092
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0713919997215271,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12697599828243256,
|
||||
"mean": 0.0767391998320818,
|
||||
"median": 0.07078400254249573,
|
||||
"min": 0.05910399928689003,
|
||||
"std": 0.018775178979463278
|
||||
},
|
||||
{
|
||||
"max": 0.11430399864912033,
|
||||
"mean": 0.07594559974968433,
|
||||
"median": 0.0713919997215271,
|
||||
"min": 0.06124800071120262,
|
||||
"std": 0.015719922150631036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08056000247597694,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1037760004401207,
|
||||
"mean": 0.07954559996724128,
|
||||
"median": 0.08056000247597694,
|
||||
"min": 0.05955199897289276,
|
||||
"std": 0.0135697420393132
|
||||
},
|
||||
{
|
||||
"max": 0.10608000308275223,
|
||||
"mean": 0.07971520014107228,
|
||||
"median": 0.07593599706888199,
|
||||
"min": 0.06028800085186958,
|
||||
"std": 0.015319849772775456
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0865279994904995,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14470399916172028,
|
||||
"mean": 0.09121599942445754,
|
||||
"median": 0.0865279994904995,
|
||||
"min": 0.06441599875688553,
|
||||
"std": 0.024893837894277456
|
||||
},
|
||||
{
|
||||
"max": 0.12438400089740753,
|
||||
"mean": 0.08531199917197227,
|
||||
"median": 0.08031999692320824,
|
||||
"min": 0.06364800035953522,
|
||||
"std": 0.01878029830059533
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07135999947786331,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11753600090742111,
|
||||
"mean": 0.07606079950928687,
|
||||
"median": 0.07135999947786331,
|
||||
"min": 0.05843200162053108,
|
||||
"std": 0.01755519771639284
|
||||
},
|
||||
{
|
||||
"max": 0.1103999987244606,
|
||||
"mean": 0.07607359997928143,
|
||||
"median": 0.07073600217700005,
|
||||
"min": 0.05721599981188774,
|
||||
"std": 0.016904445220949783
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07321599870920181,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11740799993276596,
|
||||
"mean": 0.07749119997024537,
|
||||
"median": 0.07203200086951256,
|
||||
"min": 0.05862399935722351,
|
||||
"std": 0.017701381594822835
|
||||
},
|
||||
{
|
||||
"max": 0.11382400244474411,
|
||||
"mean": 0.07733759954571724,
|
||||
"median": 0.07321599870920181,
|
||||
"min": 0.059039998799562454,
|
||||
"std": 0.017188890882557036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09025600180029869,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13980799913406372,
|
||||
"mean": 0.0950367994606495,
|
||||
"median": 0.09025600180029869,
|
||||
"min": 0.06815999746322632,
|
||||
"std": 0.022829835127539378
|
||||
},
|
||||
{
|
||||
"max": 0.14764800667762756,
|
||||
"mean": 0.09710080176591873,
|
||||
"median": 0.08720000088214874,
|
||||
"min": 0.07152000069618225,
|
||||
"std": 0.02500574954063789
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08083200082182884,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.15574400126934052,
|
||||
"mean": 0.08760640025138855,
|
||||
"median": 0.07846399769186974,
|
||||
"min": 0.07097599655389786,
|
||||
"std": 0.024487311423551025
|
||||
},
|
||||
{
|
||||
"max": 0.16284799575805664,
|
||||
"mean": 0.08963519930839539,
|
||||
"median": 0.08083200082182884,
|
||||
"min": 0.07100799679756165,
|
||||
"std": 0.026047320562445356
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.10891199856996536,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1582079976797104,
|
||||
"mean": 0.11541439890861512,
|
||||
"median": 0.10836799815297127,
|
||||
"min": 0.09062399715185165,
|
||||
"std": 0.018285136316576037
|
||||
},
|
||||
{
|
||||
"max": 0.1578879952430725,
|
||||
"mean": 0.11537599861621857,
|
||||
"median": 0.10891199856996536,
|
||||
"min": 0.0960640013217926,
|
||||
"std": 0.018246538626977286
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1703840047121048,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.19327999651432037,
|
||||
"mean": 0.1707327976822853,
|
||||
"median": 0.1703840047121048,
|
||||
"min": 0.14815999567508698,
|
||||
"std": 0.014211056022719618
|
||||
},
|
||||
{
|
||||
"max": 0.19276799261569977,
|
||||
"mean": 0.1658592015504837,
|
||||
"median": 0.16379200667142868,
|
||||
"min": 0.14560000598430634,
|
||||
"std": 0.013840249648693638
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.25539200007915497,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.2807359993457794,
|
||||
"mean": 0.25750079900026324,
|
||||
"median": 0.25539200007915497,
|
||||
"min": 0.24624000489711761,
|
||||
"std": 0.008925204570802302
|
||||
},
|
||||
{
|
||||
"max": 0.2863999903202057,
|
||||
"mean": 0.2585055992007256,
|
||||
"median": 0.255280002951622,
|
||||
"min": 0.24371199309825897,
|
||||
"std": 0.012059738582656496
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1021759994328022,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.9770879745483398,
|
||||
"mean": 0.1974783968180418,
|
||||
"median": 0.10099200159311295,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.2660581738745569
|
||||
},
|
||||
{
|
||||
"max": 0.892799973487854,
|
||||
"mean": 0.18164799660444259,
|
||||
"median": 0.1021759994328022,
|
||||
"min": 0.06435199826955795,
|
||||
"std": 0.23947520188197013
|
||||
},
|
||||
{
|
||||
"max": 0.6467199921607971,
|
||||
"mean": 0.15839359983801843,
|
||||
"median": 0.10100800171494484,
|
||||
"min": 0.06800000369548798,
|
||||
"std": 0.16617013141866102
|
||||
},
|
||||
{
|
||||
"max": 0.6725760102272034,
|
||||
"mean": 0.15686400160193442,
|
||||
"median": 0.10044800117611885,
|
||||
"min": 0.06063999980688095,
|
||||
"std": 0.17523222161300497
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12694399803876877,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.6659520268440247,
|
||||
"mean": 0.1927648030221462,
|
||||
"median": 0.12694399803876877,
|
||||
"min": 0.07609599828720093,
|
||||
"std": 0.16697784531907736
|
||||
},
|
||||
{
|
||||
"max": 0.695360004901886,
|
||||
"mean": 0.19356480240821838,
|
||||
"median": 0.11726400256156921,
|
||||
"min": 0.0796160027384758,
|
||||
"std": 0.17588096678867862
|
||||
},
|
||||
{
|
||||
"max": 0.5939840078353882,
|
||||
"mean": 0.1868800014257431,
|
||||
"median": 0.12379200011491776,
|
||||
"min": 0.07427199929952621,
|
||||
"std": 0.1463231714943902
|
||||
},
|
||||
{
|
||||
"max": 0.6635839939117432,
|
||||
"mean": 0.1874335989356041,
|
||||
"median": 0.12014400213956833,
|
||||
"min": 0.07526399940252304,
|
||||
"std": 0.16680959677760304
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09161599725484848,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12665599584579468,
|
||||
"mean": 0.09443839862942696,
|
||||
"median": 0.09124799817800522,
|
||||
"min": 0.06431999802589417,
|
||||
"std": 0.0205690155775906
|
||||
},
|
||||
{
|
||||
"max": 0.1303039938211441,
|
||||
"mean": 0.09712959825992584,
|
||||
"median": 0.09161599725484848,
|
||||
"min": 0.07648000121116638,
|
||||
"std": 0.019072048129173236
|
||||
},
|
||||
{
|
||||
"max": 0.13836799561977386,
|
||||
"mean": 0.09821119979023933,
|
||||
"median": 0.09148800000548363,
|
||||
"min": 0.0727040022611618,
|
||||
"std": 0.021314066545189116
|
||||
},
|
||||
{
|
||||
"max": 0.12992000579833984,
|
||||
"mean": 0.09272959977388381,
|
||||
"median": 0.08931199833750725,
|
||||
"min": 0.06406400352716446,
|
||||
"std": 0.021360803830354765
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08580800145864487,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13449600338935852,
|
||||
"mean": 0.08960640132427215,
|
||||
"median": 0.08299200236797333,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.020633597898445998
|
||||
},
|
||||
{
|
||||
"max": 0.14735999703407288,
|
||||
"mean": 0.09248319901525974,
|
||||
"median": 0.08580800145864487,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.025960234928829564
|
||||
},
|
||||
{
|
||||
"max": 0.13705599308013916,
|
||||
"mean": 0.08947199806571007,
|
||||
"median": 0.08460799977183342,
|
||||
"min": 0.0634239986538887,
|
||||
"std": 0.02122838946992339
|
||||
},
|
||||
{
|
||||
"max": 0.13846400380134583,
|
||||
"mean": 0.0859104000031948,
|
||||
"median": 0.08308799937367439,
|
||||
"min": 0.05974400043487549,
|
||||
"std": 0.022373631424433445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09867199882864952,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1279039978981018,
|
||||
"mean": 0.09748799875378608,
|
||||
"median": 0.09678399935364723,
|
||||
"min": 0.0655359998345375,
|
||||
"std": 0.02315719144614622
|
||||
},
|
||||
{
|
||||
"max": 0.1356479972600937,
|
||||
"mean": 0.10018239840865135,
|
||||
"median": 0.09532799944281578,
|
||||
"min": 0.06185600161552429,
|
||||
"std": 0.02330700253650042
|
||||
},
|
||||
{
|
||||
"max": 0.13142399489879608,
|
||||
"mean": 0.09778879955410957,
|
||||
"median": 0.09492799639701843,
|
||||
"min": 0.06560000032186508,
|
||||
"std": 0.02307579212430045
|
||||
},
|
||||
{
|
||||
"max": 0.1276479959487915,
|
||||
"mean": 0.09611519873142242,
|
||||
"median": 0.09867199882864952,
|
||||
"min": 0.0642239972949028,
|
||||
"std": 0.02273612181973431
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09646400064229965,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14060799777507782,
|
||||
"mean": 0.09770880043506622,
|
||||
"median": 0.09115200117230415,
|
||||
"min": 0.06950400024652481,
|
||||
"std": 0.024752645089849798
|
||||
},
|
||||
{
|
||||
"max": 0.14377599954605103,
|
||||
"mean": 0.09824960008263588,
|
||||
"median": 0.08999999985098839,
|
||||
"min": 0.07103999704122543,
|
||||
"std": 0.025403407389046027
|
||||
},
|
||||
{
|
||||
"max": 0.13680000603199005,
|
||||
"mean": 0.09993600100278854,
|
||||
"median": 0.09646400064229965,
|
||||
"min": 0.06790400296449661,
|
||||
"std": 0.022297424273985882
|
||||
},
|
||||
{
|
||||
"max": 0.1391039937734604,
|
||||
"mean": 0.09769919961690902,
|
||||
"median": 0.09601600095629692,
|
||||
"min": 0.06835199892520905,
|
||||
"std": 0.023899922100804445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08377600088715553,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1464959979057312,
|
||||
"mean": 0.09276160076260567,
|
||||
"median": 0.08377600088715553,
|
||||
"min": 0.06777600198984146,
|
||||
"std": 0.024751086465295658
|
||||
},
|
||||
{
|
||||
"max": 0.14319999516010284,
|
||||
"mean": 0.09080640003085136,
|
||||
"median": 0.080400001257658,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.023365718881708488
|
||||
},
|
||||
{
|
||||
"max": 0.1382399946451187,
|
||||
"mean": 0.09063360020518303,
|
||||
"median": 0.08193599805235863,
|
||||
"min": 0.06627199798822403,
|
||||
"std": 0.02372618650854302
|
||||
},
|
||||
{
|
||||
"max": 0.14313599467277527,
|
||||
"mean": 0.09044799953699112,
|
||||
"median": 0.08128000050783157,
|
||||
"min": 0.06652799993753433,
|
||||
"std": 0.02462486862033686
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1128000020980835,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1600320041179657,
|
||||
"mean": 0.11426239982247352,
|
||||
"median": 0.1128000020980835,
|
||||
"min": 0.07657600194215775,
|
||||
"std": 0.030240087702350687
|
||||
},
|
||||
{
|
||||
"max": 0.15881599485874176,
|
||||
"mean": 0.11206399947404862,
|
||||
"median": 0.10628800094127655,
|
||||
"min": 0.0772159993648529,
|
||||
"std": 0.029933135345483627
|
||||
},
|
||||
{
|
||||
"max": 0.15612800419330597,
|
||||
"mean": 0.10761600062251091,
|
||||
"median": 0.09860799834132195,
|
||||
"min": 0.07689599692821503,
|
||||
"std": 0.02634237020678647
|
||||
},
|
||||
{
|
||||
"max": 0.15865600109100342,
|
||||
"mean": 0.11094079986214637,
|
||||
"median": 0.10979199782013893,
|
||||
"min": 0.07583999633789062,
|
||||
"std": 0.028619217028542445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08755199983716011,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11999999731779099,
|
||||
"mean": 0.0843871995806694,
|
||||
"median": 0.08755199983716011,
|
||||
"min": 0.06332799792289734,
|
||||
"std": 0.016638543890716024
|
||||
},
|
||||
{
|
||||
"max": 0.1218239963054657,
|
||||
"mean": 0.08518079966306687,
|
||||
"median": 0.08032000064849854,
|
||||
"min": 0.06393600255250931,
|
||||
"std": 0.019900899429956945
|
||||
},
|
||||
{
|
||||
"max": 0.11849600076675415,
|
||||
"mean": 0.0843968003988266,
|
||||
"median": 0.08702399954199791,
|
||||
"min": 0.06297600269317627,
|
||||
"std": 0.017031908773433545
|
||||
},
|
||||
{
|
||||
"max": 0.12300799787044525,
|
||||
"mean": 0.0846304003149271,
|
||||
"median": 0.08139199763536453,
|
||||
"min": 0.06195199862122536,
|
||||
"std": 0.020106523698622265
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12361599877476692,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.17155200242996216,
|
||||
"mean": 0.12168959975242614,
|
||||
"median": 0.12055999785661697,
|
||||
"min": 0.09609600156545639,
|
||||
"std": 0.022565485532483928
|
||||
},
|
||||
{
|
||||
"max": 0.9246399998664856,
|
||||
"mean": 0.19978560134768486,
|
||||
"median": 0.12361599877476692,
|
||||
"min": 0.09715200215578079,
|
||||
"std": 0.24230694305662265
|
||||
},
|
||||
{
|
||||
"max": 0.9317439794540405,
|
||||
"mean": 0.20037759989500045,
|
||||
"median": 0.12327999994158745,
|
||||
"min": 0.09603200107812881,
|
||||
"std": 0.24450903278000383
|
||||
},
|
||||
{
|
||||
"max": 0.9321280121803284,
|
||||
"mean": 0.19875840097665787,
|
||||
"median": 0.12230399996042252,
|
||||
"min": 0.0950080007314682,
|
||||
"std": 0.24519252220785093
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.20030399411916733,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.3261440098285675,
|
||||
"mean": 0.20826880186796187,
|
||||
"median": 0.19974400103092194,
|
||||
"min": 0.1409280002117157,
|
||||
"std": 0.051397264855718945
|
||||
},
|
||||
{
|
||||
"max": 0.3248000144958496,
|
||||
"mean": 0.20548800230026246,
|
||||
"median": 0.1979840025305748,
|
||||
"min": 0.141184002161026,
|
||||
"std": 0.04980002399656717
|
||||
},
|
||||
{
|
||||
"max": 0.32547199726104736,
|
||||
"mean": 0.21280319690704347,
|
||||
"median": 0.20030399411916733,
|
||||
"min": 0.14127999544143677,
|
||||
"std": 0.05304243085685509
|
||||
},
|
||||
{
|
||||
"max": 0.26047998666763306,
|
||||
"mean": 0.1969312012195587,
|
||||
"median": 0.18079999834299088,
|
||||
"min": 0.14057600498199463,
|
||||
"std": 0.042174123638424224
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.2924960106611252,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.45052799582481384,
|
||||
"mean": 0.3013375997543335,
|
||||
"median": 0.28273600339889526,
|
||||
"min": 0.21334399282932281,
|
||||
"std": 0.07214223764527236
|
||||
},
|
||||
{
|
||||
"max": 0.4466240108013153,
|
||||
"mean": 0.2968191936612129,
|
||||
"median": 0.27796798944473267,
|
||||
"min": 0.21241599321365356,
|
||||
"std": 0.07400678240849741
|
||||
},
|
||||
{
|
||||
"max": 0.3830080032348633,
|
||||
"mean": 0.2946112036705017,
|
||||
"median": 0.2924960106611252,
|
||||
"min": 0.21084800362586975,
|
||||
"std": 0.05323627615746332
|
||||
},
|
||||
{
|
||||
"max": 0.4609600007534027,
|
||||
"mean": 0.3054272010922432,
|
||||
"median": 0.289792001247406,
|
||||
"min": 0.21062399446964264,
|
||||
"std": 0.07608482904865672
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
}
|
||||
],
|
||||
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
|
||||
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
|
@@ -0,0 +1,31 @@
|
||||
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation
|
||||
1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
|
@@ -0,0 +1,37 @@
|
||||
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
|
||||
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
|
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
|
||||
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 132,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v2"
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
|
142
runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py
Normal file
142
runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py
Normal file
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the profile ablation and execution-context diagnostics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--s2", type=Path, required=True)
|
||||
parser.add_argument("--routing", type=Path, required=True)
|
||||
parser.add_argument("--opprof", type=Path, required=True)
|
||||
parser.add_argument("--p1", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
s2 = json.loads(args.s2.read_text())
|
||||
routing = json.loads(args.routing.read_text())
|
||||
opprof = json.loads(args.opprof.read_text())
|
||||
p1 = json.loads(args.p1.read_text())
|
||||
|
||||
plt.rcParams.update({"font.size": 9, "axes.titlesize": 10, "axes.labelsize": 9})
|
||||
fig, axes = plt.subplots(2, 2, figsize=(13.2, 8.2), constrained_layout=True)
|
||||
|
||||
real = s2["real_scores"]
|
||||
cells = sorted(real, key=lambda cell: (-real[cell], cell))
|
||||
x = np.arange(len(cells))
|
||||
series = (
|
||||
("Real", real, "#111111", "o"),
|
||||
(
|
||||
"Old profile-only",
|
||||
s2["historical_modes"]["historical-profile-only"]["simulated_scores"],
|
||||
"#d95f02",
|
||||
"s",
|
||||
),
|
||||
(
|
||||
"vLLM 0.20 profile-only",
|
||||
s2["vllm020_profile_only"]["SLO-gated"]["simulated_scores"],
|
||||
"#7570b3",
|
||||
"^",
|
||||
),
|
||||
(
|
||||
"Frozen per-TP calibration",
|
||||
s2["historical_modes"]["historical-per-tp-calibration"]["simulated_scores"],
|
||||
"#1b9e77",
|
||||
"D",
|
||||
),
|
||||
)
|
||||
ax = axes[0, 0]
|
||||
for label, values, color, marker in series:
|
||||
ax.plot(x, [values[cell] for cell in cells], label=label, color=color, marker=marker, lw=1.7)
|
||||
ax.set_xticks(x, [cell.replace("_", "\n") for cell in cells])
|
||||
ax.set_ylabel("SLO-feasible req/s/GPU")
|
||||
ax.set_title("(a) Full 92-probe config ranking")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
ax.legend(fontsize=8, ncol=2, loc="upper right")
|
||||
|
||||
ax = axes[0, 1]
|
||||
categories = ("Actual prefill", "Actual decode", "Frontier prior")
|
||||
cv = [
|
||||
routing["phase_summary"]["prefill"]["actual"]["load_cv"]["median"],
|
||||
routing["phase_summary"]["decode"]["actual"]["load_cv"]["median"],
|
||||
routing["phase_summary"]["prefill"]["frontier_simulation"]["load_cv"]["median"],
|
||||
]
|
||||
max_ratio = [
|
||||
routing["phase_summary"]["prefill"]["actual"]["max_load_ratio"]["median"],
|
||||
routing["phase_summary"]["decode"]["actual"]["max_load_ratio"]["median"],
|
||||
routing["phase_summary"]["prefill"]["frontier_simulation"]["max_load_ratio"]["median"],
|
||||
]
|
||||
bx = np.arange(len(categories))
|
||||
width = 0.34
|
||||
bars1 = ax.bar(bx - width / 2, cv, width, label="Load CV", color="#66c2a5")
|
||||
bars2 = ax.bar(bx + width / 2, max_ratio, width, label="Max/mean load", color="#fc8d62")
|
||||
ax.bar_label(bars1, fmt="%.2f", fontsize=8)
|
||||
ax.bar_label(bars2, fmt="%.2f", fontsize=8)
|
||||
ax.set_xticks(bx, categories)
|
||||
ax.set_ylim(0, max(max_ratio) * 1.2)
|
||||
ax.set_title("(b) Per-layer MoE routing skew")
|
||||
ax.legend(fontsize=8)
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
|
||||
graph_counts = {phase: Counter() for phase in ("pure_decode", "pure_prefill", "true_mixed")}
|
||||
for cell in opprof["cells"]:
|
||||
for group in cell["groups"]:
|
||||
graph_counts[group["phase"]][group["cudagraph_runtime_mode"]] += int(group["steps"])
|
||||
ax = axes[1, 0]
|
||||
phases = tuple(graph_counts)
|
||||
bottoms = np.zeros(len(phases))
|
||||
for mode, color in (("FULL", "#1b9e77"), ("PIECEWISE", "#e6ab02"), ("NONE", "#d95f02")):
|
||||
values = np.asarray(
|
||||
[100 * graph_counts[phase][mode] / sum(graph_counts[phase].values()) for phase in phases]
|
||||
)
|
||||
ax.bar(np.arange(len(phases)), values, bottom=bottoms, label=mode, color=color)
|
||||
bottoms += values
|
||||
ax.set_xticks(np.arange(len(phases)), [phase.replace("_", " ") for phase in phases])
|
||||
ax.set_ylabel("Observed scheduler steps (%)")
|
||||
ax.set_ylim(0, 100)
|
||||
ax.set_title("(c) Real vLLM execution mode is phase-dependent")
|
||||
ax.legend(fontsize=8, ncol=3, loc="lower left")
|
||||
|
||||
ax = axes[1, 1]
|
||||
mode_order = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only")
|
||||
labels = ("Per-TP\ncalibration", "Old\nprofile-only", "vLLM 0.20\nprofile-only")
|
||||
accuracy = [100 * p1["summaries"][mode]["probe_classification"]["accuracy"] for mode in mode_order]
|
||||
mae = [100 * p1["summaries"][mode]["pass_rate_mae"] for mode in mode_order]
|
||||
px = np.arange(len(labels))
|
||||
bars1 = ax.bar(px - width / 2, accuracy, width, label="Label accuracy (higher better)", color="#1b9e77")
|
||||
bars2 = ax.bar(px + width / 2, mae, width, label="Pass-rate MAE (lower better)", color="#d95f02")
|
||||
ax.bar_label(bars1, fmt="%.1f", fontsize=8)
|
||||
ax.bar_label(bars2, fmt="%.1f", fontsize=8)
|
||||
ax.set_xticks(px, labels)
|
||||
ax.set_ylabel("Percent")
|
||||
ax.set_ylim(0, 105)
|
||||
ax.set_title("(d) Held-out P1 boundary probes (12 labels)")
|
||||
ax.legend(fontsize=8, loc="upper right")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
|
||||
fig.suptitle(
|
||||
"Qwen3-30B-A3B / vLLM 0.20 / BF16 / dash0 H20: operator provenance is not execution-context fidelity",
|
||||
fontsize=12,
|
||||
)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(args.output, dpi=180)
|
||||
fig.savefig(args.output.with_suffix(".svg"))
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare the 92 frozen S2-R-b probes with a replacement profile bundle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROFILE_KEYS = {
|
||||
"linear_op_input_file": "linear_op.csv",
|
||||
"atten_input_file": "attention.csv",
|
||||
"moe_input_file": "moe.csv",
|
||||
}
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--resolved-plan", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--profile-root", type=Path, required=True)
|
||||
parser.add_argument("--cache-root", type=Path)
|
||||
parser.add_argument("--shards", type=int, default=1)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.shards < 1:
|
||||
raise SystemExit("--shards must be positive")
|
||||
source = json.loads(args.resolved_plan.read_text())
|
||||
source_rows = [row for row in source["runs"] if row["mode"] == "uncalibrated"]
|
||||
if len(source_rows) != 92:
|
||||
raise SystemExit(f"expected 92 uncalibrated probes, found {len(source_rows)}")
|
||||
|
||||
output = args.output.resolve()
|
||||
profile_root = args.profile_root.resolve()
|
||||
cache_root = (args.cache_root or (output / "prediction-cache")).resolve()
|
||||
profile_hashes: dict[str, str] = {}
|
||||
for filename in PROFILE_KEYS.values():
|
||||
path = profile_root / filename
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing frozen profile: {path}")
|
||||
profile_hashes[str(path)] = sha256(path)
|
||||
|
||||
entries: list[dict[str, Any]] = []
|
||||
for row in source_rows:
|
||||
fixture_dir = Path(row["fixture_dir"]).resolve()
|
||||
fixture_manifest_path = fixture_dir / "fixture_manifest.json"
|
||||
fixture = json.loads(fixture_manifest_path.read_text())
|
||||
config = json.loads(Path(row["config_path"]).read_text())
|
||||
config["mode"] = "new-profile-only"
|
||||
config["config_id"] = f"{row['cell_id']}__new-profile-only"
|
||||
config["calibration"]["a_tp"] = 1.0
|
||||
knobs = config["frontier"]["knobs"]
|
||||
knobs["cache_dir"] = str(cache_root)
|
||||
knobs["no_cache"] = False
|
||||
for key, filename in PROFILE_KEYS.items():
|
||||
knobs[key] = str(profile_root / filename)
|
||||
|
||||
target_config = output / "configs" / f"{row['fixture_id']}.json"
|
||||
write_json(target_config, config)
|
||||
entries.append(
|
||||
{
|
||||
"fixture_id": row["fixture_id"],
|
||||
"cell": row["cell_id"],
|
||||
"role": f"probe-{int(row['probe_index']):02d}",
|
||||
"anchor": float(row["sampling_u"]),
|
||||
"selected_count": int(row["request_count"]),
|
||||
"probe_index": int(row["probe_index"]),
|
||||
"sampling_u": float(row["sampling_u"]),
|
||||
"tensor_parallel_size": int(row["tensor_parallel_size"]),
|
||||
"config": str(target_config),
|
||||
"fixture_manifest": str(fixture_manifest_path),
|
||||
"frontier_csv": str(fixture_dir / "frontier.csv"),
|
||||
"sidecar": str(fixture_dir / "sidecar.jsonl"),
|
||||
"calibration_scale": 1.0,
|
||||
"request_count": int(fixture["request_count"]),
|
||||
}
|
||||
)
|
||||
|
||||
base = {
|
||||
"schema": "frontier-qwen30-s2-profile-comparison-prepared.v1",
|
||||
"status": "PASS",
|
||||
"mode": "new-profile-only",
|
||||
"source": {
|
||||
"resolved_plan": str(args.resolved_plan.resolve()),
|
||||
"sha256": sha256(args.resolved_plan),
|
||||
},
|
||||
"profile_hashes": profile_hashes,
|
||||
"isolation": {
|
||||
"calibration_a_tp": 1.0,
|
||||
"prediction_cache": str(cache_root),
|
||||
"all_non_profile_knobs_inherited": True,
|
||||
},
|
||||
"suite_total_runs": len(entries),
|
||||
}
|
||||
write_json(
|
||||
output / "prepared-manifest.json",
|
||||
{**base, "expected_runs": len(entries), "entries": entries},
|
||||
)
|
||||
for shard in range(args.shards):
|
||||
shard_entries = entries[shard :: args.shards]
|
||||
write_json(
|
||||
output / f"prepared-manifest-shard-{shard:02d}-of-{args.shards:02d}.json",
|
||||
{
|
||||
**base,
|
||||
"shard": {"index": shard, "count": args.shards},
|
||||
"expected_runs": len(shard_entries),
|
||||
"entries": shard_entries,
|
||||
},
|
||||
)
|
||||
print(output / "prepared-manifest.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
360
simulator-fidelity.md
Normal file
360
simulator-fidelity.md
Normal file
@@ -0,0 +1,360 @@
|
||||
# Frontier simulator fidelity：以 config ranking 为目标的阶段性评测
|
||||
|
||||
更新日期：2026-07-16。
|
||||
|
||||
统一实验平台：所有新增与重跑实验只使用 `dash0` 的 8×NVIDIA H20。Qwen30B 的真实 P1 artifacts 也来自 `dash0`；早期文档中的主机 provenance 标注错误，本版已按实验主机和远端 artifact 路径更正。
|
||||
|
||||
## 结论摘要
|
||||
|
||||
本文统一使用同一个优化目标：**在给定 workload 和 SLO 下，最大被测试可行 offered request throughput/GPU**。我们关心 simulator 是否保持 config 的相对排序、是否覆盖真机最优集合，以及部署 simulator 所选 config 后的真实 regret；绝对 latency/throughput 误差不是首要判据。
|
||||
|
||||
当前证据不支持“Frontier 普遍找不到最优 config”这一判断：
|
||||
|
||||
- 在 Qwen3-30B mixed serving artifact 上，未经端到端校准的 Frontier 排序失败，选择 regret 为 `25.63%`；但使用独立 workload 上冻结的 per-TP calibration 后，Frontier 的候选 top set 包含真机最优 config，Kendall τ-b 为 `0.9668`，最坏 tie-break regret 为 `0.76%`。真实 serving 与本轮补测 profiles 均运行在 `dash0`。
|
||||
- 将 operator profile 更新为与真实 serving 一致的 community vLLM `0.20.0`、BF16、H20、TP1/2/4 栈后，新的 profile-only Frontier **没有恢复排序**：92 个真实 anchors 在 simulator 中全部 SLO-infeasible，12 个 config 因而全部并列，最坏 tie-break regret 为 `60.91%`。这否证了“旧 profile 版本不一致是主要原因”这一简单解释，并把问题收敛到 execution context、operator composition 与 mixed-state schema。
|
||||
- 在 Qwen3-235B-A22B-FP8 prefill-only 上，补齐 FP8/MoE profile 与 serving semantics、但不做端到端 action calibration 后，Frontier 的最优集合与真机完全一致，Spearman ρ 为 `0.9487`，20/20 个可比较非 tie config pair 同序，选择 regret 为 `0`。
|
||||
- 在 Qwen3-235B-A22B-FP8 fixed-shape mixed 上，Frontier 同样给出了与真机完全相同的 TP4 top set，Kendall τ-b=`0.8944`、worst tie-break regret=`0`。但这次成功掩盖了一个明确的机制错误：Frontier 认为同一 TP family 内四个 MNS/MBT config 完全等价，真机 TP8 capacity 却形成对角为 0.30、非对角为 0.20 req/s/GPU 的 checkerboard interaction。34 个 config-load labels 中有 10 个 false-infeasible；20 个 real non-tie pairs 中只保持 16 个方向。
|
||||
- 现有结果表明：**在已对齐的受控 case 中，绝对 capacity 有 gap、甚至 action differential 错误，都不必然妨碍 Frontier 找到当前 surface 的最优 config。** 统一 dash0 campaign 已完整覆盖 Qwen235 FP8 prefill-only 与 fixed-shape mixed，但仍不能外推到 trace-faithful mixed 或 decode-only，也不能忽略达到这种 fidelity 所需的真机 profile、KV capacity、runtime-semantic patch 或 calibration 成本。
|
||||
|
||||
因此，现阶段最准确的 research statement 不是“simulator 排序一定错误”，而是：
|
||||
|
||||
> Simulator 的 usefulness 取决于 compatibility envelope。问题是需要多少、什么类型的真机信息，才能使其 config ranking 可被信任；以及当 workload、runtime 或 execution topology 变化时，如何低成本判断该 envelope 是否仍成立。
|
||||
|
||||
## 评测口径
|
||||
|
||||
对系统 `s ∈ {real, sim}` 和 config `c`，定义：
|
||||
|
||||
```text
|
||||
capacity_s(c) = max { tested offered rate r | empirical SLO pass rate(c, r) >= 0.95 }
|
||||
```
|
||||
|
||||
所有结果使用 `capacity/实际占用 GPU 数` 作为 primary score。我们只报告实际测试到的最大可行点，不对未测试区间做连续插值，也不因观察到局部非单调就强制单调化。
|
||||
|
||||
核心指标如下：
|
||||
|
||||
- **Top set**：达到各自最大 capacity 的全部 config，保留并列。
|
||||
- **Top-set hit/match**：simulator top set 是否与 real top set 有交集/完全相同。
|
||||
- **Worst tie-break regret**：若 simulator 给出多个并列最优，任取其中最差一个在真机上的损失：
|
||||
|
||||
```text
|
||||
max[c in SimTop] (RealBest - capacity_real(c)) / RealBest
|
||||
```
|
||||
|
||||
- **Rank correlation**：有并列时优先使用 Kendall τ-b；Qwen235B 已有冻结分析使用 Spearman ρ，同时报告非 tie pair direction。
|
||||
- **Pairwise direction**：两两 config 的 `> / = / <` 是否一致。只报告 non-tied accuracy 时，会同时给出可比较 pair 数量。
|
||||
|
||||
这个定义也修正了之前的口径混用：内部 `throughput proxy` 不是“满足 SLO 的最大 throughput”，不能用来决定部署 config。本文的 Qwen30B 主结论因此取代 [旧 simulator fidelity 总结](docs/simulator-fidelity-frontier-20260711.md) 中基于 throughput-proxy 的 config-selection 结论；该旧读法仍可作为 simulator 内部机制误差的诊断证据。
|
||||
|
||||
## 实验 setup
|
||||
|
||||
### Case A：Qwen3-30B-A3B mixed serving
|
||||
|
||||
| 项目 | 设置 |
|
||||
|---|---|
|
||||
| 真机 | `dash0`，8×NVIDIA H20 |
|
||||
| model | `Qwen/Qwen3-30B-A3B` |
|
||||
| serving runtime | community vLLM `0.20.0`，CUDA 12.9 环境 |
|
||||
| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` |
|
||||
| trace | `chat_w20260311_1000`，replay time scale=`0.1`，input 0--8192 tokens，output override=128 tokens，max concurrency=64 |
|
||||
| SLO | 至少 95% requests 同时满足 TTFT 阶梯阈值（input ≤4096: 2s；≤32768: 4s；其余 6s）和 TPOT ≤50ms |
|
||||
| config surface | `TP∈{1,2,4} × MNS∈{8,16,32,64}`，MBT=8192，共 12 cells |
|
||||
| ground truth | 每个 cell 最多 8 个搜索 probes；92 个实测 anchors，共 27,583 个 request observations |
|
||||
| simulator workload | 对每个 real anchor 重放同一冻结 request cohort 的 token length、arrival、request count 与 prefix block identities；old profile-only、calibrated、vLLM 0.20 profile-only 各 92 runs，合计 276/276 成功 |
|
||||
| primary objective | SLO-feasible request throughput/GPU |
|
||||
|
||||
我们区分两个 Frontier 条件：
|
||||
|
||||
1. **Profile-only**：直接使用已有 H20 operator profiles，不用 serving E2E 数据修正时间尺度。
|
||||
2. **Frozen per-TP calibration**：使用 `TP1=0.723481, TP2=0.468089, TP4=0.352137` 三个 execution-time scales。它们在独立的 `coder_200_ts2` workload 上拟合，并在 `coder_200_ts3` 上冻结验证；本 `chat` surface 不重新拟合。该 scale 由外部 hook 在 Frontier 组成 batch execution time 后注入，并不是 Frontier 原生的 per-TP operator profiling 功能。
|
||||
|
||||
Frontier 原生 profile schema 可以按 `num_tensor_parallel_workers` 选择 TP-specific kernel rows；这与上面的 calibration 不同。第二个条件不是 zero-measurement prediction：它使用了同 model/hardware/runtime family 的真机 serving 数据，并能同时吸收 CUDA graph、fusion、routing、CPU launch 和 profile provenance 等残差。另一个限制是已有 H20 profiles 的生成栈与 ground-truth vLLM 0.20.0 并非严格同版本；per-TP calibration 吸收了相当一部分 execution-scale mismatch。
|
||||
|
||||
### Case B：Qwen3-235B-A22B-FP8 prefill-only
|
||||
|
||||
| 项目 | 设置 |
|
||||
|---|---|
|
||||
| 真机 | `dash0`，8×NVIDIA H20（97,871 MiB/GPU，全互联 NV18），driver `580.95.05` |
|
||||
| model | `Qwen/Qwen3-235B-A22B-FP8`，block-wise FP8 weights，BF16 KV cache |
|
||||
| serving runtime | community vLLM `0.10.2`；Python 3.12.3、torch 2.8.0+cu128、FlashInfer 0.3.1.post1、Transformers 4.55.2；eager execution |
|
||||
| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + 冻结的 FP8/MoE best-effort patches |
|
||||
| source trace | `thinking_w20260327_1000`；从 15,409 个 eligible requests 中固定选择 64 个 length-stratified prompts |
|
||||
| fixed cohort | seed=`2026071501`；input p50=1478、p95=18,439、max=29,385、mean=3511.55 tokens；output override=1 token |
|
||||
| offered rates | 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75, 1.00, 1.50 req/s；只缩放同一 cohort 的 arrival timeline |
|
||||
| SLO | 至少 61/64 requests 满足 `TTFT ≤ 1000ms + input_tokens/8000 tokens/s` |
|
||||
| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}`；实际 TP4 为 EP off，TP8 为 EP on |
|
||||
| serving controls | prefix cache/speculative decoding off，chunked prefill on；每个 config fresh server，第二轮反转 config/rate 顺序 |
|
||||
| primary objective | 最大被测试可行 offered rate / TP |
|
||||
|
||||
这里比较的不是 stock Frontier。为建立公平的 FP8 compatibility envelope，我们先用同一 community-vLLM 0.10.2 栈在 H20 上测量 FP8 linear/attention/MoE/collective profiles，修复了 MoE FP8 tuning-key、Qwen MoE serving plan、TP/EP-aware cache key 与 critical-lane prediction，并输入真机测得的 KV block capacity（TP4=26,101；TP8=62,351）。Frontier refined outputs 在真机 v2/v3 ranking 运行前冻结，**没有使用本 case 的 serving capacity 做 end-to-end calibration**。
|
||||
|
||||
该 case 是有意设计的 prefill mechanism-isolation workload：output 固定为 1，prefix cache 关闭，并使用固定 64-request cohort。它不保持原 trace 的 output-length distribution 和 prefix reuse，因此不能被称为 trace-faithful workload，也不能代表 mixed serving。
|
||||
|
||||
## Workload fidelity 修正
|
||||
|
||||
`thinking_w20260327_1000` 的 600 秒原始窗口包含 15,479 requests，字段包括 exact prompt、arrival、`input_length`、`output_length`、session parent/turn 和 block-size=64 的 `hash_ids`。只读审计得到：
|
||||
|
||||
| 属性 | 数值 |
|
||||
|---|---:|
|
||||
| natural offered rate | 25.798 req/s |
|
||||
| input tokens | mean=3,660.0，p50=1,491，p95=19,610.6，p99=29,944.6，max=51,827 |
|
||||
| output tokens | mean=3,924.6，p50=3,435，p95=8,945.1，p99=17,705.5，max=74,867 |
|
||||
| total context | mean=7,584.6，p50=5,317，p95=25,102.1，p99=37,110.5，max=74,973 |
|
||||
| rows with `hash_ids` | 15,479/15,479 |
|
||||
| `len(hash_ids) == ceil(input_length/64)` | 15,479/15,479 |
|
||||
| arrival-order repeated block ratio | 40.88% |
|
||||
| input+output 超过 community model 40,960 context | 72 requests |
|
||||
| output length=0 | 6 requests |
|
||||
|
||||
因此，input/output/hash filtering 不是 trace fidelity 的默认要求。此前 fixed output length 是为了分别隔离 prefill 或 decode 机制；如果用它回答 production-trace fidelity，就属于错误 benchmark。新的实验矩阵固定为：
|
||||
|
||||
| 层次 | Workload contract | Prefix contract | 回答的问题 |
|
||||
|---|---|---|---|
|
||||
| T0 fixed-shape | 固定 ISL、OSL、uniform QPS；所有 config 使用相同 request IDs/order | prefix off，使用等长但不同 token 的 prompts | 最简单条件下 execution/scheduler composition 是否匹配 |
|
||||
| T1 trace-faithful mixed | 保留 trace 的 arrival、input、output、session；不使用 completion override | real 由 exact prompt tokens 自然计算 cache key；Frontier 消费等价 `block_hash_ids` | 在真实 joint distribution 下能否保持 config rank |
|
||||
| T2 decode-only | output 保留 trace 或受控固定值；input length 表示 request 到达时已存在的 KV residency | initial-KV state 单独冻结；不得用一次 1-token prefill 冒充 | KV residency、decode batching、TP/DP/EP coupling 下能否保持 rank |
|
||||
|
||||
T1 中只允许两类有记录的排除：模型 context 上限不支持的 72 requests，以及 API 无法形成 completion 的 6 个 zero-output rows；二者不重叠，最终 eligible universe 为 15,401/15,479 requests（99.50%）。排除后 input/output/total 的均值分别为 3,575.0/3,823.3/7,398.2 tokens，p95 分别为 18,887/8,768/23,697，说明没有把 workload 人为变成 short-request case。其余请求不能因为“跑得慢”或“方便形成 cohort”被选择性移除。capacity search 只通过 trace 已有、同 session 共享的 `sampling_u` 做与长度无关的 session-coherent thinning；每个被选 request 的 arrival、input、output、prompt、hash 和相对次序均保持不变。完整 eligible universe 是抽样母体和最终 workload 声明，不用 length-stratified cohort 代替。
|
||||
|
||||
全量 tokenizer audit 进一步验证了 15,401/15,401 个 eligible prompts、共 55,057,919 tokens 的实际 community-Qwen token 数与 trace `input_length` 完全一致。在 867,538 个 source blocks 上，`hash_id` 与 `(parent_hash_id, 64-token chunk)` 的双向映射也没有发现冲突。因此 exact prompt 和 source hash 足以保留该 trace 的 prefix-equivalence relation，不需要用 synthetic length-only prompt 替代。
|
||||
|
||||
trace 的 source hash block size 为 64，而 community vLLM 0.10.2 的 CUDA KV block size 不能直接设置为 64。下一轮实验冻结 real 与 Frontier 的 block size 都为 16：从 exact prompt token IDs 重新生成 block-16 parent/content identities，并用 source `hash_ids` 验证每四个完整 block-16 所对应的 64-token equality/reuse relation。tokenized length、request order、cache query/hit/allocated blocks 仍必须在两侧逐请求核对；上面的离线 audit 不能代替 runtime counter parity。
|
||||
|
||||
## 总体结果
|
||||
|
||||
| Case / simulator 条件 | configs | real optimum | simulator top set | top-set 结论 | rank / pairwise | worst regret | 判断 |
|
||||
|---|---:|---|---|---|---|---:|---|
|
||||
| Qwen30 mixed / old profile-only | 12 | TP2,MNS32 | TP4,MNS32/64 | miss | τ-b=0.000；exact sign=37.88% | 25.63% | 排序错误 |
|
||||
| Qwen30 mixed / vLLM 0.20 profile-only | 12 | TP2,MNS32 | 全部 12 configs | 不可辨识 | τ-b=0.000；exact sign=7.58% | 60.91% | 同栈 raw profile 仍不足 |
|
||||
| Qwen30 mixed / frozen per-TP calibration | 12 | TP2,MNS32 | TP2,MNS32/64 | hit，非 exact | τ-b=0.9668；exact sign=93.94% | 0.76% | dash0 |
|
||||
| Qwen235 FP8 prefill / best-effort | 8 | TP4,MBT16K,MNS64/128 | 与 real 完全相同 | exact match | ρ=0.9487；non-tied 20/20 | 0 | 足以选最优 config |
|
||||
| Qwen235 FP8 fixed-shape mixed / frozen full profile | 8 | 四个 TP4 configs | 与 real 完全相同 | exact match | τ-b=0.8944；exact sign=24/28；real non-tie=16/20 | 0 | 选对 topology；漏掉 TP8 MNS×MBT interaction |
|
||||
|
||||
最重要的区别是：Qwen30 的 high-fidelity 结果依赖 per-TP E2E calibration；Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。二者都不能表述为“拿 stock Frontier 零成本预测即可”。
|
||||
|
||||
## Case A baseline 结果：Qwen30 mixed serving
|
||||
|
||||
|
||||
|
||||
图中 config 按真机 capacity 从高到低排列。上图显示 profile-only Frontier 选错 TP family；下图显示冻结 per-TP calibration 后，simulator 基本恢复了有用排序。
|
||||
|
||||
| Config | Real | Frontier profile-only | Frontier calibrated | Calibrated − real |
|
||||
|---|---:|---:|---:|---:|
|
||||
| TP1, MNS8 | 2.1000 | 1.1000 | 1.7167 | -0.3833 |
|
||||
| TP1, MNS16 | 2.3500 | 1.1000 | 2.3833 | +0.0333 |
|
||||
| TP1, MNS32 | 2.2833 | 1.1000 | 2.3833 | +0.1000 |
|
||||
| TP1, MNS64 | 2.2833 | 1.1000 | 2.3833 | +0.1000 |
|
||||
| TP2, MNS8 | 2.2750 | 0.0000 | 1.7417 | -0.5333 |
|
||||
| TP2, MNS16 | 2.2750 | 1.1917 | 2.3000 | +0.0250 |
|
||||
| **TP2, MNS32** | **3.2833** | 0.0000 | **3.7500** | +0.4667 |
|
||||
| TP2, MNS64 | 3.2583 | 0.0000 | **3.7500** | +0.4917 |
|
||||
| TP4, MNS8 | 1.2833 | 0.0000 | 1.3208 | +0.0375 |
|
||||
| TP4, MNS16 | 2.4417 | 0.0000 | 2.5000 | +0.0583 |
|
||||
| TP4, MNS32 | 2.4417 | **1.3208** | 2.5000 | +0.0583 |
|
||||
| TP4, MNS64 | 2.4417 | **1.3208** | 2.5000 | +0.0583 |
|
||||
|
||||
单位均为 SLO-feasible req/s/GPU。`0.0000` 表示该 config 在复用的 tested anchors 中没有被 Frontier 判为 SLO-feasible，不代表硬件在零负载下也无法运行。
|
||||
|
||||
相对排序方面：
|
||||
|
||||
- Profile-only 的 top set 为 TP4/MNS32、TP4/MNS64；两者真机 capacity 都是 2.4417，而 real optimum TP2/MNS32 是 3.2833，因此 regret 为 `1−2.4417/3.2833=25.63%`。
|
||||
- Calibrated Frontier 把 TP2/MNS32 和 TP2/MNS64 判为并列。前者是真机唯一最优，后者低 `0.025 req/s/GPU`；因此 optimistic regret=0，worst tie-break regret=`0.76%`。
|
||||
- Calibrated absolute error 并不是一个全局常数：TP2/MNS8 被低估 0.5333，而 TP2/MNS32/64 被高估约 0.47--0.49。**选对 config 不代表 simulator 已正确解释每个 state/action 的性能。**
|
||||
- 92 个 anchor-level SLO 判定中，calibrated Frontier 仍有 21 个 false-feasible、7 个 false-infeasible；aggregate top set 正确部分来自 cell boundary 上误差抵消。因此该条件足以做本 surface 的 near-optimal selection，但不能替代真机 SLO certification。
|
||||
|
||||
### 同栈 vLLM 0.20 per-TP profile 消融
|
||||
|
||||
为了判断上述差异是否只是旧 operator profile 与真实 serving 版本不一致，我们在 `dash0` 上使用 ground-truth 相同的 community vLLM `0.20.0` source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`、Qwen3-30B-A3B BF16 与 H20，重新测量 TP1/2/4 profiles。Frontier code、92 个 trace fixtures、SLO、KV capacity、config surface、communication backend 与随机种子保持不变；这轮只替换 compute profiles，不使用本 surface 的 serving latency 拟合 scale。
|
||||
|
||||
| Profile component | 冻结行数 | 测量与使用方式 |
|
||||
|---|---:|---|
|
||||
| attention + KV | 132 | 102 个 pure prefill/decode rows；30 个 true-mixed rows由同 TP pure profile 比例做 total-conserving split |
|
||||
| fused mixed diagnostic | 30 | 保存 vLLM FA3 实测 fused total；不作为 Frontier 的第三个 target |
|
||||
| linear / norm / RoPE / router | 36 | vLLM 0.20 CUDA operator |
|
||||
| MoE | 72 | TP1/2/4 local-shard shapes × 12 token sizes × uniform/random/hotset8 routing |
|
||||
| all-reduce | 24 | TP2/4 实测；本 base comparison 不注入，用来诊断 communication，保持历史 analytical CC backend 不变 |
|
||||
|
||||
attention、linear 和 MoE 的 `TP` 字段改变的是单张 H20 上的 local-shard shape；它们不是一次 distributed full-engine graph replay。只有 all-reduce 使用真实 TP2/4 ranks 测量。该限制是实验设计的一部分：本消融回答“把相同 runtime 的 isolated operator profiles 换进去是否足够”，而不是预先假设 isolated profile 已等价于真实 execution counterfactual。
|
||||
|
||||
|
||||
|
||||
完整 92-probe 结果如下。`0` 表示复用的 anchors 中没有一个被 simulator 判为 SLO-feasible；它既不表示零负载硬件 capacity，也不能被解释成“因为真实最优 config 位于并列集合中所以 top-set hit”。当所有 config 都并列时 simulator 没有选择信息，故 primary 指标是 worst tie-break regret。
|
||||
|
||||
| Frontier 条件 | Sim top set | τ-b | Exact pair sign | Worst regret | Anchor labels：agree / false feasible / false infeasible |
|
||||
|---|---|---:|---:|---:|---:|
|
||||
| old profile-only | TP4,MNS32/64 | 0.0000 | 37.88% | 25.63% | 37 / 0 / 55 |
|
||||
| vLLM 0.20 profile-only | 全部 12 configs | 0.0000 | 7.58% | 60.91% | 30 / 0 / 62 |
|
||||
| frozen per-TP calibration | TP2,MNS32/64 | 0.9668 | 93.94% | 0.76% | 64 / 21 / 7 |
|
||||
|
||||
若忽略 SLO，仅按 simulator 完成 throughput proxy 排序，新 profile 的 top-1 是 TP1/MNS64，真机 regret 仍为 `30.46%`，τ-b=`0.3940`，non-tied pair concordance=`70.49%`。这只是 failure diagnosis，不能替代本文定义的 SLO-feasible capacity objective。两路 CPU shard 都是 46/46 PASS，合计 simulator runtime 为 2,394.9 s（0.665 CPU-hour）；所以结果不是 crash、missing row 或未完成运行造成的。
|
||||
|
||||
#### 为什么相同 operator provenance 仍不等于 execution fidelity
|
||||
|
||||
证据把 mismatch 收敛到以下三个相互耦合的层面：
|
||||
|
||||
1. **Execution regime 随 scheduler state 改变。** 6 个真实 P1 cells 共记录 161,161 个 model steps。151,471 个 pure-decode steps 全部使用 FULL CUDA graph；224 个 pure-prefill steps 中 192 个为 NONE、30 个 PIECEWISE、仅 2 个 FULL；9,466 个 true-mixed steps 中 8,623 个为 NONE、842 个 PIECEWISE、仅 1 个 FULL。本轮 isolated profiles 则关闭 CUDA graph，Frontier comparison 也固定 `decode_cuda_graph_mode=none`。同一个 nominal operator 在不同 phase/state 下并不是同一个 launch/fusion regime。
|
||||
2. **Mixed attention schema 不保真。** vLLM 0.20 的 FA3 对 true-mixed batch 执行一个 fused varlen call；Frontier 必须分别得到 `attn_prefill` 与 `attn_decode_in_mixed`。不做 split 时 simulator 在第一个 mixed batch 明确报 missing prediction；采用 pure-profile ratio split 虽保持 `prefill + decode = fused total`，但 split 是兼容性归因而非真实观测。scheduler 使用这个虚构分解推进后续状态，误差会累积。
|
||||
3. **MoE routing 不是固定先验。** 对 exact trace 的 8 个 requests、48 层 native routing capture，实际 median load CV 在 prefill/decode 分别为 `1.076/1.107`，median max/mean expert load 为 `6.12/5.64`；Frontier 固定的 per-layer Uniform prior 分别只有 `0.485` 与 `1.84`。实际与 Frontier expert allocation 的逐层 Pearson median 接近 0（prefill `-0.017`、decode `-0.024`），而 Frontier 在所有 phase 复用相同 allocation，prefill/decode correlation 被固定为 1.0；真实 median correlation 为 `0.667`。8-request capture 是 mechanism probe，不是 workload population estimate，但足以否证“固定 routing vector 等价于该 trace execution”的假设。
|
||||
|
||||
更直接地，在同一个 `tp1_mns64` fixture、`batch_id=0/layer_id=0`、两条 trajectory 尚未分叉时，新 profile 的 layer component sum 已是旧 profile 的 `2.70×`（1.148→3.101 ms）：MoE grouped GEMM 为 `3.76×`、RoPE `7.39×`、attention prefill `2.67×`。这说明版本更新没有产生一个小而一致的 correction；不同 components 的误差方向和幅度不同，随后又通过 scheduler queue、batch composition、graph mode 与 routing 反馈到下一个 state。
|
||||
|
||||
因此本实验否证 H1：**Frontier 原生支持按 `num_tensor_parallel_workers` 选择 per-TP profile rows，而且 attention/MoE kernel 也确实可以逐 TP 测量；真正缺失的是 scheduler state 与 execution regime 条件化的 joint counterfactual。** TP 和 MNS 不能拆开：TP 改变 shard、collective、replica capacity、routing placement 与 graph shape，MNS 改变 batching/queue evolution；两者共同决定 phase mixture 和 execution mode。历史 per-TP calibration 是 composition 之后的外部 E2E scale，因此能吸收这些残差，但它不是机制解释，也不是 Frontier 原生 profiling。
|
||||
|
||||
下一轮不应继续盲目加 isolated rows，而应依次做可证伪的 context ablation：按 FULL pure-decode、NONE/PIECEWISE prefill/mixed 建立 graph-conditioned step profiles；保留 fused mixed layer/step total 而不是要求不可观测的拆分；用 trace-conditioned per-layer/request/phase routing histogram 替换固定先验；再单独注入 measured collective。每次只改变一个条件，最终仍在完整 `TP×MNS` joint surface 上以 SLO ranking 验证，frozen calibration 只作为可达到的上界。
|
||||
|
||||
本次 accepted profile jobs 使用约 0.118 H20-GPU-hour；包括 smoke 和失败 attempt 的完整 campaign 为 0.410 H20-GPU-hour。冻结 profile、manifest、condensed artifacts 与作图脚本位于 `runs/frontier-qwen30-vllm020-profile-v1/`。
|
||||
|
||||
## Case B 结果：Qwen235 FP8 prefill-only
|
||||
|
||||
|
||||
|
||||
| TP | EP | MNS | MBT | Real | Frontier | Frontier − real |
|
||||
|---:|---:|---:|---:|---:|---:|---:|
|
||||
| 4 | off | 64 | 8192 | 0.05000 | 0.03750 | -0.01250 |
|
||||
| 4 | off | 128 | 8192 | 0.05000 | 0.03750 | -0.01250 |
|
||||
| 4 | off | 64 | 16384 | **0.07500** | **0.06250** | -0.01250 |
|
||||
| 4 | off | 128 | 16384 | **0.07500** | **0.06250** | -0.01250 |
|
||||
| 8 | on | 64 | 8192 | 0.05625 | 0.05000 | -0.00625 |
|
||||
| 8 | on | 128 | 8192 | 0.05625 | 0.05000 | -0.00625 |
|
||||
| 8 | on | 64 | 16384 | 0.05625 | 0.05625 | 0.00000 |
|
||||
| 8 | on | 128 | 16384 | 0.05625 | 0.05625 | 0.00000 |
|
||||
|
||||
单位均为 SLO-feasible req/s/GPU。这里 Frontier 对 capacity 有明显、且依赖 topology/action family 的绝对 gap：
|
||||
|
||||
- TP4/MBT8K 与 TP4/MBT16K 都低估 0.0125 req/s/GPU，但正确预测了 MBT8K→16K 的相对增益。
|
||||
- TP8/MBT8K 低估 0.00625，而 TP8/MBT16K 恰好匹配；Frontier 因而预测 MBT 在 TP8 上有收益，真机中四个 TP8 config 实际并列。
|
||||
- 尽管这个 action differential 是错的，TP4/MBT16K 相对其余 config 有足够 margin，所以 simulator top set 仍与 real top set 完全相同，最终 regret=0。
|
||||
|
||||
细化边界实验中，Frontier/real 的 33 个 config-load labels 有 6 个 false-infeasible；这些误差在 fresh-server v3 复测中稳定重复。因此这个 case 的结论应严格限定为：**Frontier 是有效的 config ranker / conservative screener，但不是精确的 capacity estimator，也没有完全学对 topology-dependent MBT effect。**
|
||||
|
||||
## Case C 结果：Qwen235 FP8 fixed-shape mixed
|
||||
|
||||
2026-07-16 在 `dash0` GPU 0--3 上完成了 community vLLM 0.10.2、TP4/DP1、MNS64、MBT8192、eager、BF16 KV、prefix/spec/CUDA graph off 的 pilot。所有请求实际 usage 和 streaming token count 都严格为 ISL=2,048、OSL=128：
|
||||
|
||||
| load | requests | TTFT mean/p95 | TPOT mean/p95 | 40 ms joint SLO |
|
||||
|---|---:|---:|---:|---:|
|
||||
| single | 1 | 586.2 / 586.2 ms | 136.2 / 136.2 ms | 0/1 |
|
||||
| concurrency=2 | 2 | 310.2 / 434.3 ms | 124.3 / 124.9 ms | 0/2 |
|
||||
|
||||
Frontier 使用与成功 prefill ranking 相同的 frozen source fingerprint、TP4 KV blocks=26,101 和 block size=16。第一次运行完成 prefill 并提交第一个 decode token后，在纯 decode batch 抛出 `attention decode prediction cache not found for cluster monolithic`。初始化日志给出直接原因：`no standard decode rows`，而 final attention CSV 的 726 rows 全部是 prefill rows。该 crash 没有被记为 SLO fail，也没有使用 dummy prediction。
|
||||
|
||||
随后在 `dash0` 单张 H20 上用同一 community-vLLM/FlashInfer 栈补测 TP4、batch `{1,2}`、KV `{2048,2176}` 的 4 个 CUDA-event decode points。profile-closed Frontier smoke 精确完成了相同请求：
|
||||
|
||||
| load | Real TTFT | Frontier TTFT | Real TPOT | Frontier TPOT |
|
||||
|---|---:|---:|---:|---:|
|
||||
| single | 586.2 ms | 267.7 ms | 136.2 ms | 88.3 ms |
|
||||
| concurrency=2 | 186.1--434.3 ms | 470.7 ms | 123.8--124.9 ms | 90.5 ms |
|
||||
|
||||
因此 missing-row blocker 已关闭，但 absolute error 明显且不是固定比例；这正是需要 config surface 而不能只看单点 latency 的原因。steady-QPS 下还会出现 prefill+decode true-mixed batches，最小 4-row profile 不足以支持完整 sweep。完整 closure 另外测量了 TP4/TP8 各 81 个 standard-decode points 和 108 个 true-mixed points，覆盖 decode batch 到 128、KV 2,048--2,175；合并 root 共 1,104 rows。Frontier 已确认用 108 个 TP-specific samples 训练 `attn_decode_in_mixed`，没有使用本 case 的 E2E latency calibration。
|
||||
|
||||
完整 8 configs × 8 rates simulator surface 已在任何真机 surface cell 之前冻结，manifest SHA256 为 `801aa36451c8647f71cc87011144622d2203786e82f189ed1375d964399b106a`。64/64 cells 都有 64 个 finite、非负、shape-exact request records，累计 simulator CPU time 为 11,675.2 s。冻结结果如下，单位为最大被测试可行 req/s/GPU：
|
||||
|
||||
| config family | TPOT 40 ms | TPOT 120 ms | TPOT 150 ms | TPOT 180 ms |
|
||||
|---|---:|---:|---:|---:|
|
||||
| TP4，任意 MNS/MBT | none | 0.20 | 0.40 | 0.60 |
|
||||
| TP8，任意 MNS/MBT | none | 0.05 | 0.15 | 0.20 |
|
||||
|
||||
Frontier 在同一 TP family 内给四个 MNS/MBT config 生成了逐 request 完全相同的结果，即它预测这些 controls 在当前 load range 都不 binding；150 ms sensitivity 下则预测 TP4 family 的 per-GPU capacity 是 TP8 的 2.67×。这两个判断均已用下面的完整真机 surface 检验。
|
||||
|
||||
第一次真机 surface attempt 还暴露了一个 measurement-contract 问题：一个 fresh server 内连续测多个 rates 会把前一 anchor 的 execution warm state 带入后一 anchor。两个 TP4/MNS64 config 在正向 `0.10→1.60→2.40` 的 1.60 点分别只有 36/64、41/64 requests 通过 150 ms SLO；反向先运行 2.40 后，同一 1.60 点都变成 64/64。逐段看，正向 1.60 的前 24 requests TPOT 从约 234 ms 递减到 169 ms，后半段才稳定到 114--134 ms；反向 1.60 从第一个 request 起就在 113--128 ms。这个 attempt 整体排除，不能用“两个 round 都通过”的保守聚合掩盖 real 与 simulator 的 state contract 不同。
|
||||
|
||||
修正后的 ground-truth contract 对每个 `(config, rate, round)` 独立启动 server，并在目标 rate 下丢弃 `min(32, max(4, ceil(rate×20)))` 个 exact-shape warmup requests。anchor-isolation smoke 在两个 config、两个 fresh-server repeats 上均为 64/64 completed、0 failure、150 ms SLO 64/64；四次 p95 TPOT 为 127.8--138.0 ms，p95 TTFT 为 295.8--305.5 ms。只有这个通过 gate 的 contract 会进入最终 real capacity/rank。
|
||||
|
||||
### 完整评测 setup
|
||||
|
||||
| 项目 | 设置 |
|
||||
|---|---|
|
||||
| 真机 | `dash0`，8×NVIDIA H20（97,871 MiB/GPU、driver 580.95.05、NV18）；TP4 cells 各独占 4 GPU，TP8 cells 独占全部 8 GPU |
|
||||
| model/runtime | `Qwen/Qwen3-235B-A22B-FP8`；community vLLM 0.10.2、Python 3.12.3、torch 2.8.0+cu128、FlashInfer 0.3.1.post1；eager、FP8 weights、BF16 KV |
|
||||
| workload | 64 个不同 token 的固定长度 requests；ISL=2,048、OSL=128、uniform QPS、prefix/speculative decoding/CUDA graph off |
|
||||
| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}`；TP4 为 EP off，TP8 为 EP on |
|
||||
| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + frozen best-effort patches；1,104-row attention profile；无本 case E2E calibration |
|
||||
| simulator anchors | 每个 config 都运行 `{0.10,0.20,0.40,0.80,1.20,1.60,2.40,3.20}` req/s；64/64 cells 在真机 surface 前冻结 |
|
||||
| real anchors | 围绕冻结 simulator boundary 运行并自适应补齐相邻点；共 34 config-rate cells × 2 independent rounds |
|
||||
| 每个 real anchor | fresh vLLM server；先按目标 rate 丢弃 `min(32,max(4,ceil(rate×20)))` 个 warmup requests，再从空 scheduler queue 测 64 requests |
|
||||
| primary SLO | ≥61/64 requests 同时满足 TTFT≤1,256 ms、TPOT≤150 ms；两个 rounds 均通过才判定 feasible |
|
||||
| primary objective | 最大被测试可行 offered req/s / 实际占用 GPU 数 |
|
||||
|
||||
150 ms 是看到最低负载真实 TPOT 已达 136.2 ms 后公开记录的 post-pilot amendment；原 40 ms 阈值仍保留，但所有 config 在最低 anchor 都不可行。120 ms sensitivity 的 lower boundary 尚未补齐，180 ms sensitivity 在最高 3.20 anchor 仍有 censored cells，因此本文只对 150 ms primary SLO 报告完整 ranking。
|
||||
|
||||
### 完整结果
|
||||
|
||||
|
||||
|
||||
| TP | MNS | MBT | Real | Frontier | Real boundary：pass → fail（两个 rounds） |
|
||||
|---:|---:|---:|---:|---:|---|
|
||||
| 4 | 64 | 8192 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 57/64,59/64 |
|
||||
| 4 | 64 | 16384 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 62/64,57/64 |
|
||||
| 4 | 128 | 8192 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 53/64,55/64 |
|
||||
| 4 | 128 | 16384 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 52/64,54/64 |
|
||||
| 8 | 64 | 8192 | 0.30 | 0.15 | 2.40: 64/64,64/64 → 3.20: 46/64,48/64 |
|
||||
| 8 | 64 | 16384 | 0.20 | 0.15 | 1.60: 64/64,64/64 → 2.40: 45/64,53/64 |
|
||||
| 8 | 128 | 8192 | 0.20 | 0.15 | 1.60: 64/64,64/64 → 2.40: 40/64,56/64 |
|
||||
| 8 | 128 | 16384 | 0.30 | 0.15 | 2.40: 64/64,64/64 → 3.20: 50/64,46/64 |
|
||||
|
||||
Real 与 Frontier 单位均为 SLO-feasible req/s/GPU；boundary 的 offered rate 单位为 system req/s。所有 8 个 real boundaries 都由相邻 lattice points 闭合。四个 TP4 configs 是 real 与 simulator 的共同、完全相同的 top set，因此 optimistic/worst tie-break regret 都是 0。Kendall τ-b=`0.8944`；28 个 config pairs 中 exact `>/< /=` sign 命中 24 个，20 个 real non-tie pairs 中命中 16 个方向。34 个实际测量的 config-load labels 中，Frontier 命中 24 个，有 10 个 false-infeasible、0 个 false-feasible。
|
||||
|
||||
这个结果同时给出一个比“absolute error”更重要的反例。TP8 真机在 2.40 system req/s 上的 feasibility matrix 是：
|
||||
|
||||
```text
|
||||
MBT 8K MBT 16K
|
||||
MNS 64 pass fail
|
||||
MNS 128 fail pass
|
||||
```
|
||||
|
||||
换成 capacity，`ΔMBT | MNS64 = -0.10`，而 `ΔMBT | MNS128 = +0.10 req/s/GPU`，difference-in-differences 为 `+0.20 req/s/GPU`。也就是说，**MBT 的作用方向取决于 MNS；它们不能被看成两个独立、可加的 controls。** Frontier 对四个 TP8 cells 都预测 0.15，interaction 恰好为 0。相同四个 configs 在 1.60 及以下都通过，差异只在持续高负载下出现，说明问题不只是单个 operator 在空载下的固定时间偏差，而发生在 batching/scheduling state 与 execution topology 的组合上。现有数据还不能区分具体是 batch composition、MoE EP communication、memory/KV pressure，还是 runtime 的其他动态状态；下一步应对此边界做 scheduler batch trace、per-step kernel/NCCL timeline 与 KV/queue counters，而不是先给出未经验证的根因。
|
||||
|
||||
因此本 case 的严格结论是：**Frontier 足以在这个 8-config surface 上选择全局最优 topology，但不足以解释或优化 TP8 family 内的 MNS/MBT controls。** 成功来自 TP4-vs-TP8 的真实 margin 足够大，能够容忍 TP4 33.3%、TP8 25%--50% 的 capacity 低估和内部 action-ordering error；它不是 simulator 已获得 scheduling/execution 深层认知的证据。
|
||||
|
||||
接受的 ground-truth 共 18 个 fleet jobs、68 个 fresh-server anchors、4,352 个 measured requests 和 1,664 个 warmup requests，消耗 36.26 H20-GPU-hours；从第一个 accepted job 到最后一个完成的墙钟跨度为 5.74 小时。增量 full-attention closure 在单张 H20 上运行 12 秒（0.0033 H20-GPU-hours），但它复用了此前已有的 726-row prefill profile，因此这个数字不是构建全部 compatibility envelope 的总成本。另有共享 GPU/endpoint 的污染 attempts 和暴露 cross-anchor warm-state leakage 的 diagnostic attempts 被整目录隔离，不进入上述统计。Frontier 64-cell surface 消耗 11,675.2 CPU-seconds。完整 hash、逐 anchor input 与排除 ledger 见本节末的复现链接。
|
||||
|
||||
完整证据见 [T0 smoke report](runs/frontier-multicase-sufficiency-v1/t0-smoke-report.md) 与 [预注册协议](runs/frontier-multicase-sufficiency-v1/protocol.md)。
|
||||
|
||||
原预注册的 TPOT 40 ms 因最低负载已不可行而失去 capacity-ranking 可辨识性；本文已把 150 ms 明确记录为 post-pilot protocol amendment，并保留 40 ms 全部失败的 sensitivity 结果，没有把 SLO change 隐藏成预注册结论。
|
||||
|
||||
## 尚不能纳入 simulator ranking 的 case
|
||||
|
||||
### 原 internal-runtime Qwen235 prefill-only
|
||||
|
||||
原实验有 8/8 valid real cells，但使用不同的 serving/runtime contract，real top set 是四个 TP8 config。它没有与本文 best-effort Frontier 使用同一个 community-vLLM FP8 profile/serving contract，不能和新 simulator 数值直接拼接。本文的 community-vLLM Case B 是目前可辩护的 aligned prefill comparison。
|
||||
|
||||
### 原 Qwen235 decode-only
|
||||
|
||||
真实 surface 有 7/8 valid cells，且 capacity 搜索区间大量重叠：经验 top set 为 5/8，而考虑 bracket uncertainty 时 8/8 都可能最优。更关键的是，该 case 依赖 initial KV state、DecodeBenchConnector、EAGLE3、FP8 KV、DeepEP/NVSHMEM 与 decode CUDA graphs；Frontier 当前没有等价的 initial-KV/EP8 execution contract。因此现在给出 simulator-vs-real rank 或 regret 都会把 contract mismatch 误写成 fidelity 结果。
|
||||
|
||||
下一步不再用 decode-dominant mixed workload 回避问题，而是在 community vLLM 0.10.2 与 Frontier 两侧增加显式、相同的 initial-KV contract：request 到达时分配与 input length 对应的 KV blocks、标记 prefill complete，再从第一步 decode 开始调度。首先关闭 speculative decode、prefix cache 和 decode CUDA graph，建立 fixed-shape decode-only baseline；随后按单变量顺序启用 trace lengths/prefix、EP8、EAGLE3 和 FULL_DECODE_ONLY graph。每一层都重新报告 ranking，不能把后一层的 mismatch 归因给前一层。
|
||||
|
||||
## 当前判断与下一步 gate
|
||||
|
||||
当前证据的结论是：
|
||||
|
||||
1. **绝对 gap 不是否决 simulator 的理由。** Qwen235 中 11%--50% 的 capacity 低估仍可保持 zero-regret config selection。
|
||||
2. **但 rank 成功不能证明模型机制正确。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互；Qwen30 还有 28/92 anchor SLO labels 错误。只是当前 decision margin 足以容忍这些 residual。
|
||||
3. **alignment 是结果的一部分。** Qwen30 需要 per-TP E2E calibration 才能恢复排序；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。
|
||||
4. **目前不能建立“simulator 全局选优普遍失败”的 premise。** 受控 prefill、fixed-shape mixed 和历史 mixed surface 上，经过 alignment 的 Frontier 已经能选到最优或 0.76% 内的近最优 config。更有证据的 research premise 是：aggregate selection success 会掩盖 action interaction 与 state-transition model 的错误；若要声称全局选优失败，仍必须在 dash0 的 trace-faithful mixed、decode-only、prefix cache 或跨 topology case 上得到稳定反例。
|
||||
|
||||
后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。
|
||||
|
||||
## 数据与复现
|
||||
|
||||
- 图表输入：[data.json](docs/assets/simulator-fidelity/data.json)
|
||||
- 画图脚本：[plot_simulator_fidelity.py](scripts/plot_simulator_fidelity.py)
|
||||
- Qwen30 audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/qwen30-baseline/report.md)
|
||||
- Qwen30 aligned metrics：[metrics.json](runs/frontier-slo-alignment-v0/results/metrics.json)
|
||||
- Qwen235 fixed-cohort comparison：[v2_refined_comparison.json](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json)
|
||||
- Qwen235 full report：[report.md](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md)
|
||||
- Fixed-shape mixed comparison：[comparison.json](runs/frontier-multicase-sufficiency-v1/results/t0-final/comparison.json)
|
||||
- Fixed-shape mixed capacity table：[capacity.csv](runs/frontier-multicase-sufficiency-v1/results/t0-final/capacity.csv)
|
||||
- Frozen Frontier surface：[frontier_surface_frozen.json](runs/frontier-multicase-sufficiency-v1/artifacts/frontier-t0-surface-v1/frontier_surface_frozen.json)
|
||||
- Executed real plan：[real-plan-executed.json](runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan-executed.json)
|
||||
- Real exclusions/acceptance ledger：[exclusions.json](runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/exclusions.json)
|
||||
- Fixed-shape / trace-faithful / decode-only 分阶段协议：[protocol.md](runs/frontier-multicase-sufficiency-v1/protocol.md)
|
||||
- Trace tokenizer/hash contract audit：[trace-contract-audit.json](runs/frontier-multicase-sufficiency-v1/trace-contract-audit.json)
|
||||
- Trace contract audit reproducer：[audit_trace_contract.py](runs/frontier-multicase-sufficiency-v1/audit_trace_contract.py)
|
||||
- T0 real smoke 与 Frontier decode-profile failure：[t0-smoke-report.md](runs/frontier-multicase-sufficiency-v1/t0-smoke-report.md)
|
||||
- 尚未对齐 case 的 real-only audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/ground-truth/report.md)
|
||||
|
||||
重新生成 PNG/SVG：
|
||||
|
||||
```bash
|
||||
python3 scripts/plot_simulator_fidelity.py
|
||||
```
|
||||
Reference in New Issue
Block a user