Evaluate vLLM 0.20 profiles against Frontier
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare Frontier operator predictions at the identical initial scheduler state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PATTERN = re.compile(
|
||||
r"\[OP-TRACE\]\[MONOLITHIC\]\[(?:ATTENTION|MOE)\]\[([^]]+)\] "
|
||||
r"batch_id=(\d+), layer_id=(\d+), predicted_time_ms=([0-9.eE+-]+)"
|
||||
)
|
||||
COMPONENTS = (
|
||||
"input_layernorm",
|
||||
"attn_pre_proj",
|
||||
"attn_rope",
|
||||
"attn_kv_cache_save",
|
||||
"attn_prefill",
|
||||
"attn_decode",
|
||||
"attn_post_proj",
|
||||
"post_attention_layernorm",
|
||||
"moe_gating_linear",
|
||||
"moe_gating_routing_topk",
|
||||
"moe_shuffling",
|
||||
"moe_grouped_gemm",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--old", type=Path, required=True)
|
||||
parser.add_argument("--new", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def first_state(path: Path) -> dict[str, float]:
|
||||
values: dict[str, set[float]] = {}
|
||||
with path.open(errors="replace") as handle:
|
||||
for line in handle:
|
||||
match = PATTERN.search(line)
|
||||
if match and int(match[2]) == 0 and int(match[3]) == 0:
|
||||
values.setdefault(match[1], set()).add(float(match[4]))
|
||||
ambiguous = {name: sorted(items) for name, items in values.items() if len(items) != 1}
|
||||
if ambiguous:
|
||||
raise ValueError(f"ambiguous initial predictions in {path}: {ambiguous}")
|
||||
result = {name: next(iter(items)) for name, items in values.items()}
|
||||
result.setdefault("attn_decode", 0.0)
|
||||
missing = sorted(set(COMPONENTS) - set(result))
|
||||
if missing:
|
||||
raise ValueError(f"missing initial predictions in {path}: {missing}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
old = first_state(args.old)
|
||||
new = first_state(args.new)
|
||||
rows = []
|
||||
for component in COMPONENTS:
|
||||
rows.append(
|
||||
{
|
||||
"component": component,
|
||||
"historical_profile_ms": old[component],
|
||||
"vllm020_profile_ms": new[component],
|
||||
"new_over_old": new[component] / old[component]
|
||||
if old[component] != 0
|
||||
else None,
|
||||
}
|
||||
)
|
||||
old_total = sum(old[name] for name in COMPONENTS)
|
||||
new_total = sum(new[name] for name in COMPONENTS)
|
||||
output = {
|
||||
"schema": "frontier-initial-op-trace-delta.v1",
|
||||
"comparison_contract": {
|
||||
"fixture": "fidelity_p1_tp1_mns64_low1",
|
||||
"scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge",
|
||||
"calibration_a_tp": 1.0,
|
||||
},
|
||||
"layer_component_sum_ms": {
|
||||
"historical_profile": old_total,
|
||||
"vllm020_profile": new_total,
|
||||
"new_over_old": new_total / old_total,
|
||||
},
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyze real, calibrated, old-profile, and new-profile P1 probe outcomes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MODES = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--controller-state", type=Path, required=True)
|
||||
parser.add_argument("--calibrated", type=Path, required=True)
|
||||
parser.add_argument("--old-profile-only", type=Path, required=True)
|
||||
parser.add_argument("--new-profile-only", type=Path, required=True)
|
||||
parser.add_argument("--output-json", type=Path, required=True)
|
||||
parser.add_argument("--output-csv", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_results(path: Path) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
payload = json.loads(path.read_text())
|
||||
if payload["status"] != "PASS" or len(payload["results"]) != 12:
|
||||
raise ValueError(f"expected 12 passing simulator probes in {path}")
|
||||
return {(row["cell"], row["role"]): row for row in payload["results"]}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
controller = json.loads(args.controller_state.read_text())
|
||||
real: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for cell, value in controller["cells"].items():
|
||||
tp = int(value["tp"])
|
||||
for run in value["runs"]:
|
||||
if run["role"] in ("low1", "high1"):
|
||||
real[(cell, run["role"])] = {
|
||||
**run,
|
||||
"tp": tp,
|
||||
"offered_req_s_per_gpu": int(run["selected_count"]) / 60 / tp,
|
||||
}
|
||||
if len(real) != 12:
|
||||
raise ValueError(f"expected 12 real P1 probes, found {len(real)}")
|
||||
|
||||
modes = {
|
||||
"historical-calibrated": load_results(args.calibrated),
|
||||
"historical-profile-only": load_results(args.old_profile_only),
|
||||
"vllm020-profile-only": load_results(args.new_profile_only),
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
summaries: dict[str, Any] = {}
|
||||
for mode in MODES:
|
||||
predicted = modes[mode]
|
||||
false_feasible = 0
|
||||
false_infeasible = 0
|
||||
pass_errors: list[float] = []
|
||||
capacity_lower_bounds: dict[str, float] = {}
|
||||
for key in sorted(real):
|
||||
real_row = real[key]
|
||||
sim_row = predicted[key]
|
||||
scorer = sim_row["scorer"]
|
||||
sim_feasible = bool(scorer["slo"]["feasible"])
|
||||
real_feasible = bool(real_row["feasible"])
|
||||
false_feasible += int(sim_feasible and not real_feasible)
|
||||
false_infeasible += int(real_feasible and not sim_feasible)
|
||||
pass_error = abs(float(scorer["slo"]["pass_rate"]) - float(real_row["pass_rate"]))
|
||||
pass_errors.append(pass_error)
|
||||
rows.append(
|
||||
{
|
||||
"mode": mode,
|
||||
"cell": key[0],
|
||||
"role": key[1],
|
||||
"real_feasible": real_feasible,
|
||||
"sim_feasible": sim_feasible,
|
||||
"real_pass_rate": float(real_row["pass_rate"]),
|
||||
"sim_pass_rate": float(scorer["slo"]["pass_rate"]),
|
||||
"pass_rate_absolute_error": pass_error,
|
||||
"offered_req_s_per_gpu": float(real_row["offered_req_s_per_gpu"]),
|
||||
"sim_throughput_req_s_per_gpu": float(
|
||||
scorer["throughput_requests_per_second_per_gpu"]
|
||||
),
|
||||
}
|
||||
)
|
||||
if sim_feasible:
|
||||
capacity_lower_bounds[key[0]] = max(
|
||||
capacity_lower_bounds.get(key[0], 0.0),
|
||||
float(real_row["offered_req_s_per_gpu"]),
|
||||
)
|
||||
agreement = 12 - false_feasible - false_infeasible
|
||||
summaries[mode] = {
|
||||
"probe_classification": {
|
||||
"agreement": agreement,
|
||||
"accuracy": agreement / 12,
|
||||
"false_feasible": false_feasible,
|
||||
"false_infeasible": false_infeasible,
|
||||
},
|
||||
"pass_rate_mae": statistics.mean(pass_errors),
|
||||
"feasible_probe_count": sum(
|
||||
bool(row["scorer"]["slo"]["feasible"]) for row in predicted.values()
|
||||
),
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
cell: capacity_lower_bounds.get(cell, 0.0)
|
||||
for cell in sorted(controller["cells"])
|
||||
},
|
||||
"rank_identifiable": bool(capacity_lower_bounds),
|
||||
}
|
||||
|
||||
output = {
|
||||
"schema": "frontier-qwen30-p1-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": 6,
|
||||
"probes_per_cell": 2,
|
||||
"roles": ["low1", "high1"],
|
||||
"reading": "held-out boundary classification, not a complete capacity sweep",
|
||||
},
|
||||
"sources": {
|
||||
"controller_state": str(args.controller_state.resolve()),
|
||||
"historical_calibrated": str(args.calibrated.resolve()),
|
||||
"historical_profile_only": str(args.old_profile_only.resolve()),
|
||||
"vllm020_profile_only": str(args.new_profile_only.resolve()),
|
||||
},
|
||||
"summaries": summaries,
|
||||
"rows": rows,
|
||||
}
|
||||
args.output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_json.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
args.output_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output_csv.open("w", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(args.output_json)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare captured vLLM expert loads with Frontier's fixed routing prior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--routing", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def gini(values: np.ndarray) -> float:
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
if values.sum() == 0:
|
||||
return 0.0
|
||||
ordered = np.sort(values)
|
||||
n = len(ordered)
|
||||
indices = np.arange(1, n + 1, dtype=np.float64)
|
||||
return float((2 * np.sum(indices * ordered) / np.sum(ordered) - (n + 1)) / n)
|
||||
|
||||
|
||||
def stats(values: np.ndarray) -> dict[str, float]:
|
||||
values = np.asarray(values, dtype=np.float64)
|
||||
mean = float(np.mean(values))
|
||||
return {
|
||||
"load_cv": float(np.std(values) / mean),
|
||||
"load_gini": gini(values),
|
||||
"max_load_ratio": float(np.max(values) / mean),
|
||||
"expert_utilization": float(np.count_nonzero(values) / len(values)),
|
||||
}
|
||||
|
||||
|
||||
def proportional_counts(total: int, ratios: np.ndarray) -> np.ndarray:
|
||||
exact = total * ratios / ratios.sum()
|
||||
counts = np.floor(exact).astype(np.int64)
|
||||
remainder = total - int(counts.sum())
|
||||
order = sorted(
|
||||
range(len(ratios)),
|
||||
key=lambda i: (-(exact[i] - counts[i]), -(ratios[i] / ratios.sum()), i),
|
||||
)
|
||||
for index in range(remainder):
|
||||
counts[order[index % len(order)]] += 1
|
||||
assert int(counts.sum()) == total
|
||||
return counts
|
||||
|
||||
|
||||
def correlation(left: np.ndarray, right: np.ndarray) -> float:
|
||||
value = float(np.corrcoef(left, right)[0, 1])
|
||||
return value if math.isfinite(value) else 0.0
|
||||
|
||||
|
||||
def distribution(values: list[float]) -> dict[str, float]:
|
||||
return {
|
||||
"min": min(values),
|
||||
"median": statistics.median(values),
|
||||
"max": max(values),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
routing = json.loads(args.routing.read_text())
|
||||
phases = routing["phases"]
|
||||
layer_count = len(phases["prefill"]["per_layer"])
|
||||
expert_count = len(phases["prefill"]["per_layer"][0]["counts"])
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for phase in ("prefill", "decode"):
|
||||
for layer, actual in enumerate(phases[phase]["per_layer"]):
|
||||
rng = np.random.RandomState(args.seed + layer)
|
||||
ratios = rng.uniform(0.1, 1.0, expert_count)
|
||||
synthetic = proportional_counts(int(actual["total_routed_tokens"]), ratios)
|
||||
synthetic_stats = stats(synthetic)
|
||||
for name in synthetic_stats:
|
||||
if not math.isclose(
|
||||
stats(np.asarray(actual["counts"]))[name],
|
||||
float(actual[name]),
|
||||
rel_tol=0,
|
||||
abs_tol=1e-12,
|
||||
):
|
||||
raise ValueError(f"captured metric mismatch: {phase} layer {layer} {name}")
|
||||
rows.append(
|
||||
{
|
||||
"phase": phase,
|
||||
"layer": layer,
|
||||
"total_routed_tokens": int(actual["total_routed_tokens"]),
|
||||
"actual": {name: float(actual[name]) for name in synthetic_stats},
|
||||
"frontier_simulation": synthetic_stats,
|
||||
"actual_vs_frontier_pearson": correlation(
|
||||
np.asarray(actual["counts"], dtype=np.float64), synthetic
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
phase_summary: dict[str, Any] = {}
|
||||
for phase in ("prefill", "decode"):
|
||||
selected = [row for row in rows if row["phase"] == phase]
|
||||
phase_summary[phase] = {
|
||||
"token_count": int(phases[phase]["token_count"]),
|
||||
"actual": {
|
||||
name: distribution([row["actual"][name] for row in selected])
|
||||
for name in selected[0]["actual"]
|
||||
},
|
||||
"frontier_simulation": {
|
||||
name: distribution([row["frontier_simulation"][name] for row in selected])
|
||||
for name in selected[0]["frontier_simulation"]
|
||||
},
|
||||
"actual_vs_frontier_pearson": distribution(
|
||||
[row["actual_vs_frontier_pearson"] for row in selected]
|
||||
),
|
||||
}
|
||||
|
||||
phase_correlations = []
|
||||
for layer in range(layer_count):
|
||||
prefill = np.asarray(phases["prefill"]["per_layer"][layer]["counts"])
|
||||
decode = np.asarray(phases["decode"]["per_layer"][layer]["counts"])
|
||||
phase_correlations.append(correlation(prefill, decode))
|
||||
|
||||
output = {
|
||||
"schema": "frontier-routing-mismatch.v1",
|
||||
"source": str(args.routing.resolve()),
|
||||
"frontier_contract": {
|
||||
"mode": "simulation",
|
||||
"seed": args.seed,
|
||||
"allocation": "per-layer fixed Uniform(0.1, 1.0), normalized once and reused for every batch and phase",
|
||||
"layer_count": layer_count,
|
||||
"expert_count": expert_count,
|
||||
},
|
||||
"phase_summary": phase_summary,
|
||||
"actual_prefill_vs_decode_pearson": distribution(phase_correlations),
|
||||
"frontier_prefill_vs_decode_pearson": 1.0,
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Score a replacement-profile S2 sweep against the frozen real oracle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--shard-metrics", type=Path, action="append", required=True)
|
||||
parser.add_argument("--ground-truth", type=Path, required=True)
|
||||
parser.add_argument("--historical-metrics", type=Path, required=True)
|
||||
parser.add_argument("--historical-analyzer", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_module(path: Path):
|
||||
spec = importlib.util.spec_from_file_location("simfid_s2_analyzer", path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def ranking(scores: dict[str, float]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"rank": index + 1, "cell": cell, "score": score}
|
||||
for index, (cell, score) in enumerate(
|
||||
sorted(scores.items(), key=lambda item: (-item[1], item[0]))
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def robust_loao(
|
||||
analyzer: Any,
|
||||
runs: list[dict[str, Any]],
|
||||
mode: str,
|
||||
reading: str,
|
||||
real_scores: dict[str, float],
|
||||
) -> dict[str, Any]:
|
||||
"""Historical LOAO with an explicit null range for all-tied concordance."""
|
||||
anchors = sorted(
|
||||
{(row["cell_id"], int(row["probe_index"])) for row in runs},
|
||||
key=lambda item: (analyzer.CELL_ORDER.index(item[0]), item[1]),
|
||||
)
|
||||
replicates = []
|
||||
undefined = []
|
||||
for cell, probe in anchors:
|
||||
scores, detail = analyzer.cell_scores(
|
||||
runs, mode, reading, removed=(cell, probe)
|
||||
)
|
||||
if scores is None:
|
||||
undefined.append(
|
||||
{"removed_cell_id": cell, "removed_probe_index": probe, **detail}
|
||||
)
|
||||
continue
|
||||
metric = analyzer.rank_metrics(real_scores, scores)
|
||||
replicates.append(
|
||||
{
|
||||
"removed_cell_id": cell,
|
||||
"removed_probe_index": probe,
|
||||
"top1_optimistic_regret": metric["top1"]["optimistic_regret"],
|
||||
"top1_worst_case_regret": metric["top1"]["worst_case_regret"],
|
||||
"top5_minimum_exact_five_overlap": metric["top5"]["minimum_exact_five_overlap"],
|
||||
"top5_maximum_exact_five_overlap": metric["top5"]["maximum_exact_five_overlap"],
|
||||
"top5_optimistic_regret": metric["top5"]["optimistic_regret"],
|
||||
"top5_worst_case_regret": metric["top5"]["worst_case_regret"],
|
||||
"tau_b": metric["kendall_tau_b"]["tau_b"],
|
||||
"pairwise_exact_sign_accuracy": metric["pairwise_direction"]["exact_sign_accuracy"],
|
||||
"pairwise_non_tied_concordance": metric["pairwise_direction"]["non_tied_concordance"],
|
||||
"trap_reproduced": metric["named_interactions"]["trap_reproduced"],
|
||||
"tp2_mns32_unique_global_best": metric["named_interactions"]["tp2_mns32_unique_global_best"],
|
||||
}
|
||||
)
|
||||
scalar_keys = (
|
||||
"top1_optimistic_regret",
|
||||
"top1_worst_case_regret",
|
||||
"top5_minimum_exact_five_overlap",
|
||||
"top5_maximum_exact_five_overlap",
|
||||
"top5_optimistic_regret",
|
||||
"top5_worst_case_regret",
|
||||
"tau_b",
|
||||
"pairwise_exact_sign_accuracy",
|
||||
"pairwise_non_tied_concordance",
|
||||
)
|
||||
ranges = {}
|
||||
for key in scalar_keys:
|
||||
values = [row[key] for row in replicates if row[key] is not None]
|
||||
ranges[key] = {
|
||||
"min": min(values) if values else None,
|
||||
"max": max(values) if values else None,
|
||||
}
|
||||
return {
|
||||
"replicate_count": len(anchors),
|
||||
"defined_replicates": len(replicates),
|
||||
"undefined_replicates": len(undefined),
|
||||
"undefined": undefined,
|
||||
"ranges": ranges,
|
||||
"trap_reproduced_count": sum(row["trap_reproduced"] for row in replicates),
|
||||
"tp2_mns32_unique_global_best_count": sum(
|
||||
row["tp2_mns32_unique_global_best"] for row in replicates
|
||||
),
|
||||
"replicates": replicates,
|
||||
"range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
analyzer = load_module(args.historical_analyzer)
|
||||
ground = json.loads(args.ground_truth.read_text())
|
||||
cells = {str(cell["cell_id"]): cell for cell in ground["cells"]}
|
||||
if set(cells) != set(analyzer.CELL_ORDER):
|
||||
raise ValueError("ground-truth cells differ from the frozen 3x4 surface")
|
||||
|
||||
result_rows: list[dict[str, Any]] = []
|
||||
sources = []
|
||||
for path in args.shard_metrics:
|
||||
payload = json.loads(path.read_text())
|
||||
if payload["status"] != "PASS":
|
||||
raise ValueError(f"shard did not pass: {path}")
|
||||
sources.append({"path": str(path.resolve()), "runs": len(payload["results"])})
|
||||
result_rows.extend(payload["results"])
|
||||
if len(result_rows) != 92:
|
||||
raise ValueError(f"expected 92 replacement-profile probes, found {len(result_rows)}")
|
||||
|
||||
seen: set[tuple[str, int]] = set()
|
||||
runs = []
|
||||
for row in result_rows:
|
||||
cell_id = str(row["cell"])
|
||||
probe_index = int(row["probe_index"])
|
||||
key = (cell_id, probe_index)
|
||||
if key in seen:
|
||||
raise ValueError(f"duplicate probe {key}")
|
||||
seen.add(key)
|
||||
real_probe = cells[cell_id]["probe_history"][probe_index]
|
||||
if not math.isclose(
|
||||
float(real_probe["sampling_u"]), float(row["sampling_u"]), rel_tol=0, abs_tol=1e-15
|
||||
):
|
||||
raise ValueError(f"sampling_u mismatch for {key}")
|
||||
if int(real_probe["request_count"]) != int(row["selected_count"]):
|
||||
raise ValueError(f"request count mismatch for {key}")
|
||||
runs.append(
|
||||
{
|
||||
**row,
|
||||
"cell_id": cell_id,
|
||||
"mode": "vllm020-profile-only",
|
||||
"probe_index": probe_index,
|
||||
"sampling_u": float(row["sampling_u"]),
|
||||
"request_count": int(row["selected_count"]),
|
||||
"tensor_parallel_size": int(row["tensor_parallel_size"]),
|
||||
"real_anchor": {
|
||||
"feasible": bool(real_probe["feasible"]),
|
||||
"pass_rate": float(real_probe["pass_rate"]),
|
||||
"request_count": int(real_probe["request_count"]),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
historical = json.loads(args.historical_metrics.read_text())
|
||||
real_scores = {cell: float(value) for cell, value in historical["real_scores"].items()}
|
||||
analyses = {}
|
||||
for reading in analyzer.READINGS:
|
||||
scores, detail = analyzer.cell_scores(runs, "vllm020-profile-only", reading)
|
||||
if scores is None:
|
||||
raise ValueError(f"undefined {reading} scores: {detail}")
|
||||
analysis = {
|
||||
"reading": reading,
|
||||
"simulated_scores": scores,
|
||||
"ranking": ranking(scores),
|
||||
"cell_score_details": detail["cells"],
|
||||
"metrics": analyzer.rank_metrics(real_scores, scores),
|
||||
"loao": robust_loao(
|
||||
analyzer, runs, "vllm020-profile-only", reading, real_scores
|
||||
),
|
||||
}
|
||||
if reading == "SLO-gated":
|
||||
analysis["false_feasibility"] = analyzer.false_feasibility(
|
||||
runs, "vllm020-profile-only"
|
||||
)
|
||||
analyses[reading] = analysis
|
||||
|
||||
historical_modes = {}
|
||||
for label, key in (
|
||||
("historical-profile-only", "uncalibrated/SLO-gated"),
|
||||
("historical-per-tp-calibration", "frozen-calibrated/SLO-gated"),
|
||||
):
|
||||
value = historical["analyses"][key]
|
||||
historical_modes[label] = {
|
||||
"simulated_scores": value["simulated_scores"],
|
||||
"ranking": ranking(value["simulated_scores"]),
|
||||
"metrics": value["metrics"],
|
||||
"false_feasibility": value["false_feasibility"],
|
||||
}
|
||||
|
||||
output = {
|
||||
"schema": "frontier-qwen30-s2-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": len(analyzer.CELL_ORDER),
|
||||
"probes": len(runs),
|
||||
"trace_horizon_seconds": 60,
|
||||
"calibration_a_tp": 1.0,
|
||||
},
|
||||
"sources": {
|
||||
"replacement_profile_shards": sources,
|
||||
"ground_truth": str(args.ground_truth.resolve()),
|
||||
"historical_metrics": str(args.historical_metrics.resolve()),
|
||||
},
|
||||
"real_scores": real_scores,
|
||||
"historical_modes": historical_modes,
|
||||
"vllm020_profile_only": analyses,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"comparison_contract": {
|
||||
"calibration_a_tp": 1.0,
|
||||
"fixture": "fidelity_p1_tp1_mns64_low1",
|
||||
"scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge"
|
||||
},
|
||||
"layer_component_sum_ms": {
|
||||
"historical_profile": 1.148334,
|
||||
"new_over_old": 2.700194368537377,
|
||||
"vllm020_profile": 3.1007249999999997
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"component": "input_layernorm",
|
||||
"historical_profile_ms": 0.010619,
|
||||
"new_over_old": 1.955833882663151,
|
||||
"vllm020_profile_ms": 0.020769
|
||||
},
|
||||
{
|
||||
"component": "attn_pre_proj",
|
||||
"historical_profile_ms": 0.204389,
|
||||
"new_over_old": 1.0199276869107439,
|
||||
"vllm020_profile_ms": 0.208462
|
||||
},
|
||||
{
|
||||
"component": "attn_rope",
|
||||
"historical_profile_ms": 0.018908,
|
||||
"new_over_old": 7.394171779141103,
|
||||
"vllm020_profile_ms": 0.139809
|
||||
},
|
||||
{
|
||||
"component": "attn_kv_cache_save",
|
||||
"historical_profile_ms": 0.014226,
|
||||
"new_over_old": 1.427386475467454,
|
||||
"vllm020_profile_ms": 0.020306
|
||||
},
|
||||
{
|
||||
"component": "attn_prefill",
|
||||
"historical_profile_ms": 0.107759,
|
||||
"new_over_old": 2.670709639102071,
|
||||
"vllm020_profile_ms": 0.287793
|
||||
},
|
||||
{
|
||||
"component": "attn_decode",
|
||||
"historical_profile_ms": 0.0,
|
||||
"new_over_old": null,
|
||||
"vllm020_profile_ms": 0.0
|
||||
},
|
||||
{
|
||||
"component": "attn_post_proj",
|
||||
"historical_profile_ms": 0.134153,
|
||||
"new_over_old": 0.967559428413826,
|
||||
"vllm020_profile_ms": 0.129801
|
||||
},
|
||||
{
|
||||
"component": "post_attention_layernorm",
|
||||
"historical_profile_ms": 0.011481,
|
||||
"new_over_old": 0.9000087100426792,
|
||||
"vllm020_profile_ms": 0.010333
|
||||
},
|
||||
{
|
||||
"component": "moe_gating_linear",
|
||||
"historical_profile_ms": 0.011802,
|
||||
"new_over_old": 2.2642772411455687,
|
||||
"vllm020_profile_ms": 0.026723
|
||||
},
|
||||
{
|
||||
"component": "moe_gating_routing_topk",
|
||||
"historical_profile_ms": 0.009662,
|
||||
"new_over_old": 2.2922790312564687,
|
||||
"vllm020_profile_ms": 0.022148
|
||||
},
|
||||
{
|
||||
"component": "moe_shuffling",
|
||||
"historical_profile_ms": 0.030431,
|
||||
"new_over_old": 0.0,
|
||||
"vllm020_profile_ms": 0.0
|
||||
},
|
||||
{
|
||||
"component": "moe_grouped_gemm",
|
||||
"historical_profile_ms": 0.594904,
|
||||
"new_over_old": 3.756204362384519,
|
||||
"vllm020_profile_ms": 2.234581
|
||||
}
|
||||
],
|
||||
"schema": "frontier-initial-op-trace-delta.v1"
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
mode,cell,role,real_feasible,sim_feasible,real_pass_rate,sim_pass_rate,pass_rate_absolute_error,offered_req_s_per_gpu,sim_throughput_req_s_per_gpu
|
||||
historical-calibrated,tp1_mns64,high1,False,True,0.24581005586592178,0.9888268156424581,0.7430167597765364,2.9833333333333334,2.8823345959582403
|
||||
historical-calibrated,tp1_mns64,low1,True,True,1.0,1.0,0.0,2.033333333333333,1.9806287931485522
|
||||
historical-calibrated,tp1_mns8,high1,False,False,0.20670391061452514,0.16759776536312848,0.03910614525139666,2.9833333333333334,2.162661799726284
|
||||
historical-calibrated,tp1_mns8,low1,True,True,1.0,0.9754098360655737,0.024590163934426257,2.033333333333333,1.9806313066623409
|
||||
historical-calibrated,tp2_mns64,high1,True,True,1.0,1.0,0.0,2.875,2.809583898291986
|
||||
historical-calibrated,tp2_mns64,low1,True,True,1.0,1.0,0.0,1.9583333333333333,1.911740196570019
|
||||
historical-calibrated,tp2_mns8,high1,False,False,0.20238095238095238,0.15476190476190477,0.047619047619047616,2.8,2.083217332903509
|
||||
historical-calibrated,tp2_mns8,low1,True,True,1.0,0.9868995633187773,0.013100436681222738,1.9083333333333334,1.8637394408880168
|
||||
historical-calibrated,tp4_mns16,high1,False,False,0.10666666666666667,0.2693333333333333,0.16266666666666663,3.125,2.6829294600763345
|
||||
historical-calibrated,tp4_mns16,low1,False,True,0.6196078431372549,1.0,0.3803921568627451,2.125,2.0865667413849516
|
||||
historical-calibrated,tp4_mns64,high1,True,True,1.0,1.0,0.0,3.125,3.0718260341342285
|
||||
historical-calibrated,tp4_mns64,low1,True,True,1.0,1.0,0.0,2.125,2.0967062495193307
|
||||
historical-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.24581005586592178,0.0,2.9833333333333334,2.82546810606982
|
||||
historical-profile-only,tp1_mns64,low1,True,False,1.0,0.7377049180327869,0.2622950819672131,2.033333333333333,1.9427723213024963
|
||||
historical-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.09497206703910614,0.111731843575419,2.9833333333333334,1.5728783582441022
|
||||
historical-profile-only,tp1_mns8,low1,True,False,1.0,0.1721311475409836,0.8278688524590164,2.033333333333333,1.5629164685645112
|
||||
historical-profile-only,tp2_mns64,high1,True,False,1.0,0.2608695652173913,0.7391304347826086,2.875,2.7121507155597584
|
||||
historical-profile-only,tp2_mns64,low1,True,False,1.0,0.8382978723404255,0.16170212765957448,1.9583333333333333,1.849101894293262
|
||||
historical-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.041666666666666664,0.16071428571428573,2.8,0.9859710483293953
|
||||
historical-profile-only,tp2_mns8,low1,True,False,1.0,0.0611353711790393,0.9388646288209607,1.9083333333333334,0.9771135206177947
|
||||
historical-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.028,0.07866666666666668,3.125,0.9604809617478992
|
||||
historical-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.058823529411764705,0.5607843137254902,2.125,0.9495221159963836
|
||||
historical-profile-only,tp4_mns64,high1,True,False,1.0,0.0026666666666666666,0.9973333333333333,3.125,2.281094127221267
|
||||
historical-profile-only,tp4_mns64,low1,True,False,1.0,0.4627450980392157,0.5372549019607843,2.125,1.9793960278798877
|
||||
vllm020-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.0,0.24581005586592178,2.9833333333333334,2.074240230678028
|
||||
vllm020-profile-only,tp1_mns64,low1,True,False,1.0,0.0,1.0,2.033333333333333,1.6189525827713862
|
||||
vllm020-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.0,0.20670391061452514,2.9833333333333334,0.5271192714936628
|
||||
vllm020-profile-only,tp1_mns8,low1,True,False,1.0,0.0,1.0,2.033333333333333,0.5225514701978402
|
||||
vllm020-profile-only,tp2_mns64,high1,True,False,1.0,0.0,1.0,2.875,1.7237810220598582
|
||||
vllm020-profile-only,tp2_mns64,low1,True,False,1.0,0.0,1.0,1.9583333333333333,1.608919184557297
|
||||
vllm020-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.0,0.20238095238095238,2.8,0.39315152956277477
|
||||
vllm020-profile-only,tp2_mns8,low1,True,False,1.0,0.0,1.0,1.9083333333333334,0.3894201655965407
|
||||
vllm020-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.0,0.10666666666666667,3.125,0.43700447014112276
|
||||
vllm020-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.0,0.6196078431372549,2.125,0.43475717185302865
|
||||
vllm020-profile-only,tp4_mns64,high1,True,False,1.0,0.0,1.0,3.125,1.2598449015844073
|
||||
vllm020-profile-only,tp4_mns64,low1,True,False,1.0,0.0,1.0,2.125,1.2025831680037373
|
||||
|
@@ -0,0 +1,511 @@
|
||||
{
|
||||
"rows": [
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.7430167597765364,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9888268156424581,
|
||||
"sim_throughput_req_s_per_gpu": 2.8823345959582403
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.9806287931485522
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.03910614525139666,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.16759776536312848,
|
||||
"sim_throughput_req_s_per_gpu": 2.162661799726284
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.024590163934426257,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9754098360655737,
|
||||
"sim_throughput_req_s_per_gpu": 1.9806313066623409
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.809583898291986
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.911740196570019
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.047619047619047616,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.15476190476190477,
|
||||
"sim_throughput_req_s_per_gpu": 2.083217332903509
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 0.013100436681222738,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 0.9868995633187773,
|
||||
"sim_throughput_req_s_per_gpu": 1.8637394408880168
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.16266666666666663,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.2693333333333333,
|
||||
"sim_throughput_req_s_per_gpu": 2.6829294600763345
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.3803921568627451,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.0865667413849516
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 3.0718260341342285
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-calibrated",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": true,
|
||||
"sim_pass_rate": 1.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.0967062495193307
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.0,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.24581005586592178,
|
||||
"sim_throughput_req_s_per_gpu": 2.82546810606982
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.2622950819672131,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.7377049180327869,
|
||||
"sim_throughput_req_s_per_gpu": 1.9427723213024963
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.111731843575419,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.09497206703910614,
|
||||
"sim_throughput_req_s_per_gpu": 1.5728783582441022
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 0.8278688524590164,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.1721311475409836,
|
||||
"sim_throughput_req_s_per_gpu": 1.5629164685645112
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 0.7391304347826086,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.2608695652173913,
|
||||
"sim_throughput_req_s_per_gpu": 2.7121507155597584
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 0.16170212765957448,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.8382978723404255,
|
||||
"sim_throughput_req_s_per_gpu": 1.849101894293262
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.16071428571428573,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.041666666666666664,
|
||||
"sim_throughput_req_s_per_gpu": 0.9859710483293953
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 0.9388646288209607,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0611353711790393,
|
||||
"sim_throughput_req_s_per_gpu": 0.9771135206177947
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.07866666666666668,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.028,
|
||||
"sim_throughput_req_s_per_gpu": 0.9604809617478992
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.5607843137254902,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.058823529411764705,
|
||||
"sim_throughput_req_s_per_gpu": 0.9495221159963836
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.9973333333333333,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0026666666666666666,
|
||||
"sim_throughput_req_s_per_gpu": 2.281094127221267
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "historical-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.5372549019607843,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.4627450980392157,
|
||||
"sim_throughput_req_s_per_gpu": 1.9793960278798877
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.24581005586592178,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.24581005586592178,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 2.074240230678028
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.6189525827713862
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.9833333333333334,
|
||||
"pass_rate_absolute_error": 0.20670391061452514,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20670391061452514,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.5271192714936628
|
||||
},
|
||||
{
|
||||
"cell": "tp1_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.033333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.5225514701978402
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.875,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.7237810220598582
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9583333333333333,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.608919184557297
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.8,
|
||||
"pass_rate_absolute_error": 0.20238095238095238,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.20238095238095238,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.39315152956277477
|
||||
},
|
||||
{
|
||||
"cell": "tp2_mns8",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 1.9083333333333334,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.3894201655965407
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 0.10666666666666667,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.10666666666666667,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.43700447014112276
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns16",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 0.6196078431372549,
|
||||
"real_feasible": false,
|
||||
"real_pass_rate": 0.6196078431372549,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 0.43475717185302865
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 3.125,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "high1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.2598449015844073
|
||||
},
|
||||
{
|
||||
"cell": "tp4_mns64",
|
||||
"mode": "vllm020-profile-only",
|
||||
"offered_req_s_per_gpu": 2.125,
|
||||
"pass_rate_absolute_error": 1.0,
|
||||
"real_feasible": true,
|
||||
"real_pass_rate": 1.0,
|
||||
"role": "low1",
|
||||
"sim_feasible": false,
|
||||
"sim_pass_rate": 0.0,
|
||||
"sim_throughput_req_s_per_gpu": 1.2025831680037373
|
||||
}
|
||||
],
|
||||
"schema": "frontier-qwen30-p1-profile-ablation.v1",
|
||||
"scope": {
|
||||
"cells": 6,
|
||||
"probes_per_cell": 2,
|
||||
"reading": "held-out boundary classification, not a complete capacity sweep",
|
||||
"roles": [
|
||||
"low1",
|
||||
"high1"
|
||||
]
|
||||
},
|
||||
"sources": {
|
||||
"controller_state": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/controller-state.json",
|
||||
"historical_calibrated": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/results/metrics.json",
|
||||
"historical_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/old-profile-only-v2/results/metrics.json",
|
||||
"vllm020_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2/results/metrics.json"
|
||||
},
|
||||
"summaries": {
|
||||
"historical-calibrated": {
|
||||
"feasible_probe_count": 9,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 2.9833333333333334,
|
||||
"tp1_mns8": 2.033333333333333,
|
||||
"tp2_mns64": 2.875,
|
||||
"tp2_mns8": 1.9083333333333334,
|
||||
"tp4_mns16": 2.125,
|
||||
"tp4_mns64": 3.125
|
||||
},
|
||||
"pass_rate_mae": 0.11754094806600345,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.8333333333333334,
|
||||
"agreement": 10,
|
||||
"false_feasible": 2,
|
||||
"false_infeasible": 0
|
||||
},
|
||||
"rank_identifiable": true
|
||||
},
|
||||
"historical-profile-only": {
|
||||
"feasible_probe_count": 0,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 0.0,
|
||||
"tp1_mns8": 0.0,
|
||||
"tp2_mns64": 0.0,
|
||||
"tp2_mns8": 0.0,
|
||||
"tp4_mns16": 0.0,
|
||||
"tp4_mns64": 0.0
|
||||
},
|
||||
"pass_rate_mae": 0.44802887255544604,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.4166666666666667,
|
||||
"agreement": 5,
|
||||
"false_feasible": 0,
|
||||
"false_infeasible": 7
|
||||
},
|
||||
"rank_identifiable": false
|
||||
},
|
||||
"vllm020-profile-only": {
|
||||
"feasible_probe_count": 0,
|
||||
"p1_capacity_lower_bounds_req_s_per_gpu": {
|
||||
"tp1_mns64": 0.0,
|
||||
"tp1_mns8": 0.0,
|
||||
"tp2_mns64": 0.0,
|
||||
"tp2_mns8": 0.0,
|
||||
"tp4_mns16": 0.0,
|
||||
"tp4_mns64": 0.0
|
||||
},
|
||||
"pass_rate_mae": 0.69843078572211,
|
||||
"probe_classification": {
|
||||
"accuracy": 0.4166666666666667,
|
||||
"agreement": 5,
|
||||
"false_feasible": 0,
|
||||
"false_infeasible": 7
|
||||
},
|
||||
"rank_identifiable": false
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,469 @@
|
||||
{
|
||||
"block_size": 64,
|
||||
"cohort_output": "/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/trace-support/cohort.jsonl",
|
||||
"cohort_sha256": "9ef15a4c8843127a5c52ff8455bdeb9945ac6921f2977dd11070ed6944eed7c8",
|
||||
"full_downsampled_cohort": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 15.0,
|
||||
"p50": 38.0,
|
||||
"p75": 82.0,
|
||||
"p90": 111.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8186.0,
|
||||
"min": 71.0,
|
||||
"p10": 208.0,
|
||||
"p25": 933.0,
|
||||
"p50": 2421.0,
|
||||
"p75": 5244.0,
|
||||
"p90": 7103.0,
|
||||
"p95": 7775.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 0.20259999999998257,
|
||||
"min": 0.06279999999997088,
|
||||
"p10": 0.0917999999999708,
|
||||
"p25": 0.10240000000003846,
|
||||
"p50": 0.1163999999999703,
|
||||
"p75": 0.13009999999999167,
|
||||
"p90": 0.14210000000002765,
|
||||
"p95": 0.15119999999997447,
|
||||
"p99": 0.17959999999997933
|
||||
},
|
||||
"multi_turn_fraction": 0.140625,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.09274807669777795,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 55.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 49.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3520.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 3136.0
|
||||
},
|
||||
"rows_with_hash_ids": 512,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 25607,
|
||||
"unique_blocks": 23232
|
||||
},
|
||||
"request_count": 512,
|
||||
"sampling_u": {
|
||||
"max": 0.9979913161468553,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.10365756051317851,
|
||||
"p25": 0.22877823992809823,
|
||||
"p50": 0.4667483893139273,
|
||||
"p75": 0.7338360613813898,
|
||||
"p90": 0.8969698510725516,
|
||||
"p95": 0.9558957048665733,
|
||||
"p99": 0.991130680217395
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"Trace length/hash support does not determine dynamic decode or mixed batch shapes.",
|
||||
"Those shapes depend jointly on arrival history, SLO pressure, TP execution time, MNS, chunking, and KV eviction.",
|
||||
"MoE expert routing is not present in the trace and must be measured from model execution."
|
||||
],
|
||||
"loader_contract": {
|
||||
"completion_tokens_override": 128,
|
||||
"downsampling": "AITuner _downsample_requests before sampling_u threshold",
|
||||
"input_length_filter": {
|
||||
"max": 8192,
|
||||
"min": 0
|
||||
},
|
||||
"max_requests_per_probe": 512,
|
||||
"ordering": "arrival_s",
|
||||
"replay_time_scale": 0.1
|
||||
},
|
||||
"schema_version": "qwen30_trace_profile_support.v1",
|
||||
"study": "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/configs/examples/dash0_qwen30b_a3b_community_vllm020_noharness.json",
|
||||
"study_sha256": "cd0b9e2385751478f459faeae2cf3d12c58229866176911051421bc043bf99d4",
|
||||
"threshold_cohorts": {
|
||||
"0.125": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 127.0,
|
||||
"min": 2.0,
|
||||
"p10": 7.0,
|
||||
"p25": 15.0,
|
||||
"p50": 30.0,
|
||||
"p75": 76.0,
|
||||
"p90": 105.0,
|
||||
"p95": 111.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8110.0,
|
||||
"min": 74.0,
|
||||
"p10": 392.0,
|
||||
"p25": 922.0,
|
||||
"p50": 1864.0,
|
||||
"p75": 4844.0,
|
||||
"p90": 6686.0,
|
||||
"p95": 7054.0,
|
||||
"p99": 8110.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 6.217200000000002,
|
||||
"min": 0.10639999999998517,
|
||||
"p10": 0.16680000000001272,
|
||||
"p25": 0.30810000000001736,
|
||||
"p50": 0.6001999999999938,
|
||||
"p75": 1.1201000000000016,
|
||||
"p90": 1.8216999999999643,
|
||||
"p95": 2.419699999999967,
|
||||
"p99": 6.217200000000002
|
||||
},
|
||||
"multi_turn_fraction": 0.15151515151515152,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.10466666666666667,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 13.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 13.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 832.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 832.0
|
||||
},
|
||||
"rows_with_hash_ids": 66,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 3000,
|
||||
"unique_blocks": 2686
|
||||
},
|
||||
"request_count": 66,
|
||||
"sampling_u": {
|
||||
"max": 0.12321850600323363,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.01599560440427848,
|
||||
"p25": 0.03636425746001375,
|
||||
"p50": 0.06451196858246099,
|
||||
"p75": 0.10162658487005978,
|
||||
"p90": 0.1138971596689021,
|
||||
"p95": 0.12001874701158237,
|
||||
"p99": 0.12321850600323363
|
||||
}
|
||||
},
|
||||
"0.25": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 14.0,
|
||||
"p50": 34.0,
|
||||
"p75": 87.0,
|
||||
"p90": 107.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"p10": 197.0,
|
||||
"p25": 867.0,
|
||||
"p50": 2167.0,
|
||||
"p75": 5568.0,
|
||||
"p90": 6841.0,
|
||||
"p95": 7745.0,
|
||||
"p99": 8110.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 2.3110999999999855,
|
||||
"min": 0.07340000000003499,
|
||||
"p10": 0.11439999999997497,
|
||||
"p25": 0.14210000000002765,
|
||||
"p50": 0.3411000000000044,
|
||||
"p75": 0.5650000000000093,
|
||||
"p90": 0.8215000000000146,
|
||||
"p95": 1.088799999999992,
|
||||
"p99": 1.7577999999999996
|
||||
},
|
||||
"multi_turn_fraction": 0.11888111888111888,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.07953890489913544,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 13.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 13.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 832.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 832.0
|
||||
},
|
||||
"rows_with_hash_ids": 143,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 6940,
|
||||
"unique_blocks": 6388
|
||||
},
|
||||
"request_count": 143,
|
||||
"sampling_u": {
|
||||
"max": 0.24941730181025415,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.03095529459918134,
|
||||
"p25": 0.07431293234041883,
|
||||
"p50": 0.13301201741611407,
|
||||
"p75": 0.1994148366998705,
|
||||
"p90": 0.2292686392189136,
|
||||
"p95": 0.24107176388894497,
|
||||
"p99": 0.24671643798343437
|
||||
}
|
||||
},
|
||||
"0.5": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 2.0,
|
||||
"p25": 15.0,
|
||||
"p50": 42.0,
|
||||
"p75": 86.0,
|
||||
"p90": 109.0,
|
||||
"p95": 117.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"p10": 113.0,
|
||||
"p25": 905.0,
|
||||
"p50": 2648.0,
|
||||
"p75": 5490.0,
|
||||
"p90": 6926.0,
|
||||
"p95": 7476.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 1.2561000000000142,
|
||||
"min": 0.06749999999997236,
|
||||
"p10": 0.10019999999999563,
|
||||
"p25": 0.11840000000002249,
|
||||
"p50": 0.15819999999994394,
|
||||
"p75": 0.2599999999999483,
|
||||
"p90": 0.3867999999999938,
|
||||
"p95": 0.5083999999999804,
|
||||
"p99": 0.696700000000007
|
||||
},
|
||||
"multi_turn_fraction": 0.14130434782608695,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.08318264014466546,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 52.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 24.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3328.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 1536.0
|
||||
},
|
||||
"rows_with_hash_ids": 276,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 13825,
|
||||
"unique_blocks": 12675
|
||||
},
|
||||
"request_count": 276,
|
||||
"sampling_u": {
|
||||
"max": 0.49832600818075473,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.056494245471656455,
|
||||
"p25": 0.1313132001019981,
|
||||
"p50": 0.24403265916374703,
|
||||
"p75": 0.3706937275751198,
|
||||
"p90": 0.4530058956953501,
|
||||
"p95": 0.48171254599865826,
|
||||
"p99": 0.49639910750753047
|
||||
}
|
||||
},
|
||||
"1.0": {
|
||||
"hash_blocks_per_request": {
|
||||
"max": 128.0,
|
||||
"min": 2.0,
|
||||
"p10": 4.0,
|
||||
"p25": 15.0,
|
||||
"p50": 38.0,
|
||||
"p75": 82.0,
|
||||
"p90": 111.0,
|
||||
"p95": 122.0,
|
||||
"p99": 127.0
|
||||
},
|
||||
"input_tokens": {
|
||||
"max": 8186.0,
|
||||
"min": 71.0,
|
||||
"p10": 208.0,
|
||||
"p25": 933.0,
|
||||
"p50": 2421.0,
|
||||
"p75": 5244.0,
|
||||
"p90": 7103.0,
|
||||
"p95": 7775.0,
|
||||
"p99": 8074.0
|
||||
},
|
||||
"interarrival_s": {
|
||||
"max": 0.20259999999998257,
|
||||
"min": 0.06279999999997088,
|
||||
"p10": 0.0917999999999708,
|
||||
"p25": 0.10240000000003846,
|
||||
"p50": 0.1163999999999703,
|
||||
"p75": 0.13009999999999167,
|
||||
"p90": 0.14210000000002765,
|
||||
"p95": 0.15119999999997447,
|
||||
"p99": 0.17959999999997933
|
||||
},
|
||||
"multi_turn_fraction": 0.140625,
|
||||
"output_tokens": {
|
||||
"max": 128.0,
|
||||
"min": 128.0,
|
||||
"p10": 128.0,
|
||||
"p25": 128.0,
|
||||
"p50": 128.0,
|
||||
"p75": 128.0,
|
||||
"p90": 128.0,
|
||||
"p95": 128.0,
|
||||
"p99": 128.0
|
||||
},
|
||||
"prefix_reuse_upper_bound": {
|
||||
"any_position_reusable_block_ratio": 0.09274807669777795,
|
||||
"leading_reusable_blocks_per_request": {
|
||||
"max": 55.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 1.0,
|
||||
"p75": 13.0,
|
||||
"p90": 13.0,
|
||||
"p95": 13.0,
|
||||
"p99": 49.0
|
||||
},
|
||||
"leading_reusable_tokens_per_request": {
|
||||
"max": 3520.0,
|
||||
"min": 0.0,
|
||||
"p10": 0.0,
|
||||
"p25": 0.0,
|
||||
"p50": 64.0,
|
||||
"p75": 832.0,
|
||||
"p90": 832.0,
|
||||
"p95": 832.0,
|
||||
"p99": 3136.0
|
||||
},
|
||||
"rows_with_hash_ids": 512,
|
||||
"semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate",
|
||||
"total_blocks": 25607,
|
||||
"unique_blocks": 23232
|
||||
},
|
||||
"request_count": 512,
|
||||
"sampling_u": {
|
||||
"max": 0.9979913161468553,
|
||||
"min": 0.0017297900650431776,
|
||||
"p10": 0.10365756051317851,
|
||||
"p25": 0.22877823992809823,
|
||||
"p50": 0.4667483893139273,
|
||||
"p75": 0.7338360613813898,
|
||||
"p90": 0.8969698510725516,
|
||||
"p95": 0.9558957048665733,
|
||||
"p99": 0.991130680217395
|
||||
}
|
||||
}
|
||||
},
|
||||
"trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl",
|
||||
"trace_sha256": "f539f38eb0ee0f750e3c23ff47df6eed3faf723a25f1444d55665a85871750b9",
|
||||
"window_id": "chat_w20260311_1000"
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# 实验 EXP-SIMFID-Q30-P020：同栈 per-TP operator profile 能否恢复 Frontier ranking？
|
||||
|
||||
> **状态：** 运行中（用户已批准 5-step campaign）
|
||||
> **状态：** 完成；H1 rejected，结果支持 H2（2026-07-16）
|
||||
|
||||
## Claim 与决策
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
|
||||
## 预期产物与 review
|
||||
|
||||
- **预期数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；12-cell simulation outputs；三 baseline comparison table。
|
||||
- **Figure prototype：** `mock-profile-ablation.png`；x 为 12 configs，y 为 normalized capacity；series 为 real、old profile-only、new profile-only、calibrated。图中数值明确标为 mock，不进入结论。
|
||||
- **实际数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；92/92 new profile-only runs；old/new/calibrated comparison；scheduler graph-mode 与 native MoE routing diagnostics。
|
||||
- **Figure：** `../../docs/assets/simulator-fidelity/qwen30-vllm020-profile-ablation.png`；四个 panels 分别展示完整 ranking、routing skew、phase-dependent graph mode 和 held-out P1 boundary accuracy。图中只使用实测或冻结分析结果。
|
||||
- **人工 review：** 已批准。用户明确要求推进 5 个步骤，并要求 smoke 通过后直接完整运行。
|
||||
- **Review 意见：** 不能把 TP–MNS 耦合拆成互不相关问题；profile 只冻结 execution counterfactual 的证据，最终仍以完整 config ranking 判断。
|
||||
|
||||
@@ -32,12 +32,27 @@
|
||||
- **Code：** AITuner 当前 branch `codex/fidelity-prefix-pilot-20260714`；Frontier canonical commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e`；vLLM commit/tag `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1` / `0.20.0`。
|
||||
- **Environment：** `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；hardware/driver/package freeze 写入每个 profile artifact。
|
||||
- **产物路径：** remote `/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/`; local harvest `runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/`。
|
||||
- **已知 deviation：** 历史 real ground truth 来自 dash1；本次 profile 按用户要求在 dash0 生成。driver 也已从历史 570.133.20 漂移到 dash0 当前 580.95.05。结果只能归因于“profile stack alignment 在当前 dash0 上的效果”，不能声称严格复现旧 dash1 execution latency。
|
||||
- **已知 deviation：** 历史 real ground truth 与本次 profile 都来自 dash0；早期文档中的主机 provenance 标注错误。driver 从历史 570.133.20 漂移到当前 580.95.05，因此结果仍不能解释为严格的逐 kernel latency 复现，只能作为同一 H20 主机上 profile-stack alignment 的对照。
|
||||
|
||||
## 结果
|
||||
|
||||
- **观察事实：** 待完整 profile 与 simulation。
|
||||
- **异常：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 自带的 attention benchmark可调用实际 FlashAttention backend，但 mixed prefill+decode 在 FA3 中是一个 fused varlen call，而 Frontier schema将其拆成 `attn_prefill` 与 `attn_decode` 两项。
|
||||
- **Interpretation 与剩余 alternatives：** 这既是 profiler compatibility gap，也是潜在 representational gap。先用真实 kernel smoke 判断能否无损物化 Frontier profile；不能无损时同时保留 fused total 与 schema projection，避免误把 projection error 当 kernel error。
|
||||
- **Claim update：** unchanged。
|
||||
- **下一步：** 通过 vLLM 0.20 exact-kernel smoke；冻结 trace-derived shape support；完整 TP1/2/4 profiling；profile-only rerun。
|
||||
- **Step 1—同栈固定：** dash0 8×H20；Qwen3-30B-A3B BF16；community vLLM `0.20.0+cu129` / source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`。真实 serving 与 profiles 使用同一主机和 model family；历史 driver 570.133.20 到当前 580.95.05 的漂移保留为限制。
|
||||
- **Step 2—per-TP profiles：** profile-v2 冻结 attention 132 rows、直接测量 fused-mixed diagnostic 30 rows、linear 36 rows、MoE 72 rows、all-reduce 24 rows。attention/linear/MoE 通过单 H20 local-shard shape 表达 TP；all-reduce 使用真实 TP2/4 ranks。base comparison 保持 historical analytical CC backend，measured all-reduce 仅作独立诊断。
|
||||
- **Step 3—trace execution coverage：** 6 个 P1 cells 共 161,161 scheduler/model steps：pure decode 151,471（100% FULL graph），pure prefill 224（192 NONE、30 PIECEWISE、2 FULL），true mixed 9,466（8,623 NONE、842 PIECEWISE、1 FULL）。exact-trace 8-request native routing probe 表明实际 median max/mean expert load 为 prefill 6.12、decode 5.64，而 Frontier fixed prior 为 1.84；actual-vs-prior Pearson median 接近 0。该小 cohort 用于机制否证，不作为 routing population estimate。
|
||||
- **Step 4—冻结 profile-only：** profile-v1 因 true-mixed schema 缺少 `attn_decode_in_mixed` 在第一个 mixed batch fail。profile-v2 使用 same-TP pure prefill/decode ratio 做 total-conserving compatibility split，同时保留直接实测 fused total。两个 46-run CPU shards 均 PASS，92/92 runs 无 crash，总 simulator runtime 2,394.9 s（0.665 CPU-hour）。
|
||||
- **Step 5—ranking comparison：** real optimum 为 TP2/MNS32。old profile-only top set 为 TP4/MNS32/64，τ-b=0、worst regret=25.63%；new vLLM 0.20 profile-only 在所有 92 anchors 上都 SLO-infeasible，12 configs 全并列，τ-b=0、exact pair sign=7.58%、worst tie-break regret=60.91%；historical frozen per-TP calibration top set 为 TP2/MNS32/64，τ-b=0.9668、worst regret=0.76%。在 held-out P1 12 boundary labels 上，new profile-only accuracy 41.67%、pass-rate MAE 69.84%，也没有优于 old profile-only；calibration 分别为 83.33% 和 11.75%。
|
||||
- **Same-state delta：** 在 `tp1_mns64` 的 batch 0/layer 0、profile-dependent trajectories 尚未分叉时，new/old layer component sum 已为 2.70×；MoE grouped GEMM 3.76×、RoPE 7.39×、attention prefill 2.67×。误差不是可由一个 version correction 解释的一致 scale。
|
||||
- **异常与 schema gap：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 的 FA3 mixed prefill+decode 是一个 fused varlen call，而 Frontier schema要求 `attn_prefill` 和 `attn_decode_in_mixed` 两项。兼容 split 是归因，不是观测。
|
||||
- **Interpretation：** H1 被否证。同栈 isolated operator profile 不等价于真实 execution counterfactual。TP 与 MNS 共同改变 local shape、collective、queue/batch composition、phase mixture、routing 和 CUDA graph regime；static operator sum 在 scheduler 推进后产生 feedback amplification。历史 per-TP calibration 是 composition 之后的外部 E2E scale，能吸收这些残差，但不是 Frontier 原生 per-TP profiling 或机制解释。
|
||||
- **Claim update：** 支持 H2。新的 research target 是验证 execution-context-conditioned composition，而不是继续把更多 isolated kernel rows 当作 fidelity 的充分条件。
|
||||
- **下一步：** 分别做 graph-conditioned step profile、fused mixed total、trace-conditioned routing 和 measured collective 四个单变量消融，再回到完整 TP×MNS joint surface 评测；frozen calibration 仅作上界，不参与 mechanism fitting。
|
||||
|
||||
## 产物索引
|
||||
|
||||
- Frozen profile：`frozen/profile-v2/manifest.json`
|
||||
- Full-surface comparison：`artifacts/qwen30-s2-profile-ablation.json`
|
||||
- P1 boundary comparison：`artifacts/qwen30-p1-profile-ablation.json`
|
||||
- Scheduler/graph-mode summary：`artifacts/qwen30-p1b-opprof-summary.json`
|
||||
- Native routing mismatch：`artifacts/qwen30-routing-mismatch.json`
|
||||
- Same-state operator delta：`artifacts/qwen30-initial-op-trace-delta.json`
|
||||
- Figure script：`plot_profile_ablation.py`
|
||||
|
||||
@@ -0,0 +1,824 @@
|
||||
{
|
||||
"environment": [
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
}
|
||||
],
|
||||
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
|
||||
"rows": [
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08288000151515007,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.4872959852218628,
|
||||
"mean": 0.1310015980154276,
|
||||
"median": 0.07679999992251396,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.12790721677293843
|
||||
},
|
||||
{
|
||||
"max": 0.4402880072593689,
|
||||
"mean": 0.12842560112476348,
|
||||
"median": 0.08288000151515007,
|
||||
"min": 0.0655680000782013,
|
||||
"std": 0.11220176524616535
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0793600007891655,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12716799974441528,
|
||||
"mean": 0.07871360033750534,
|
||||
"median": 0.0759200006723404,
|
||||
"min": 0.06032000109553337,
|
||||
"std": 0.019331314939874535
|
||||
},
|
||||
{
|
||||
"max": 0.12380799651145935,
|
||||
"mean": 0.08059840016067028,
|
||||
"median": 0.0793600007891655,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.01687088356254092
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0713919997215271,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12697599828243256,
|
||||
"mean": 0.0767391998320818,
|
||||
"median": 0.07078400254249573,
|
||||
"min": 0.05910399928689003,
|
||||
"std": 0.018775178979463278
|
||||
},
|
||||
{
|
||||
"max": 0.11430399864912033,
|
||||
"mean": 0.07594559974968433,
|
||||
"median": 0.0713919997215271,
|
||||
"min": 0.06124800071120262,
|
||||
"std": 0.015719922150631036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08056000247597694,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1037760004401207,
|
||||
"mean": 0.07954559996724128,
|
||||
"median": 0.08056000247597694,
|
||||
"min": 0.05955199897289276,
|
||||
"std": 0.0135697420393132
|
||||
},
|
||||
{
|
||||
"max": 0.10608000308275223,
|
||||
"mean": 0.07971520014107228,
|
||||
"median": 0.07593599706888199,
|
||||
"min": 0.06028800085186958,
|
||||
"std": 0.015319849772775456
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0865279994904995,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14470399916172028,
|
||||
"mean": 0.09121599942445754,
|
||||
"median": 0.0865279994904995,
|
||||
"min": 0.06441599875688553,
|
||||
"std": 0.024893837894277456
|
||||
},
|
||||
{
|
||||
"max": 0.12438400089740753,
|
||||
"mean": 0.08531199917197227,
|
||||
"median": 0.08031999692320824,
|
||||
"min": 0.06364800035953522,
|
||||
"std": 0.01878029830059533
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07135999947786331,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11753600090742111,
|
||||
"mean": 0.07606079950928687,
|
||||
"median": 0.07135999947786331,
|
||||
"min": 0.05843200162053108,
|
||||
"std": 0.01755519771639284
|
||||
},
|
||||
{
|
||||
"max": 0.1103999987244606,
|
||||
"mean": 0.07607359997928143,
|
||||
"median": 0.07073600217700005,
|
||||
"min": 0.05721599981188774,
|
||||
"std": 0.016904445220949783
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07321599870920181,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11740799993276596,
|
||||
"mean": 0.07749119997024537,
|
||||
"median": 0.07203200086951256,
|
||||
"min": 0.05862399935722351,
|
||||
"std": 0.017701381594822835
|
||||
},
|
||||
{
|
||||
"max": 0.11382400244474411,
|
||||
"mean": 0.07733759954571724,
|
||||
"median": 0.07321599870920181,
|
||||
"min": 0.059039998799562454,
|
||||
"std": 0.017188890882557036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09025600180029869,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13980799913406372,
|
||||
"mean": 0.0950367994606495,
|
||||
"median": 0.09025600180029869,
|
||||
"min": 0.06815999746322632,
|
||||
"std": 0.022829835127539378
|
||||
},
|
||||
{
|
||||
"max": 0.14764800667762756,
|
||||
"mean": 0.09710080176591873,
|
||||
"median": 0.08720000088214874,
|
||||
"min": 0.07152000069618225,
|
||||
"std": 0.02500574954063789
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08083200082182884,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.15574400126934052,
|
||||
"mean": 0.08760640025138855,
|
||||
"median": 0.07846399769186974,
|
||||
"min": 0.07097599655389786,
|
||||
"std": 0.024487311423551025
|
||||
},
|
||||
{
|
||||
"max": 0.16284799575805664,
|
||||
"mean": 0.08963519930839539,
|
||||
"median": 0.08083200082182884,
|
||||
"min": 0.07100799679756165,
|
||||
"std": 0.026047320562445356
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.10891199856996536,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1582079976797104,
|
||||
"mean": 0.11541439890861512,
|
||||
"median": 0.10836799815297127,
|
||||
"min": 0.09062399715185165,
|
||||
"std": 0.018285136316576037
|
||||
},
|
||||
{
|
||||
"max": 0.1578879952430725,
|
||||
"mean": 0.11537599861621857,
|
||||
"median": 0.10891199856996536,
|
||||
"min": 0.0960640013217926,
|
||||
"std": 0.018246538626977286
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1703840047121048,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.19327999651432037,
|
||||
"mean": 0.1707327976822853,
|
||||
"median": 0.1703840047121048,
|
||||
"min": 0.14815999567508698,
|
||||
"std": 0.014211056022719618
|
||||
},
|
||||
{
|
||||
"max": 0.19276799261569977,
|
||||
"mean": 0.1658592015504837,
|
||||
"median": 0.16379200667142868,
|
||||
"min": 0.14560000598430634,
|
||||
"std": 0.013840249648693638
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.25539200007915497,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.2807359993457794,
|
||||
"mean": 0.25750079900026324,
|
||||
"median": 0.25539200007915497,
|
||||
"min": 0.24624000489711761,
|
||||
"std": 0.008925204570802302
|
||||
},
|
||||
{
|
||||
"max": 0.2863999903202057,
|
||||
"mean": 0.2585055992007256,
|
||||
"median": 0.255280002951622,
|
||||
"min": 0.24371199309825897,
|
||||
"std": 0.012059738582656496
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1021759994328022,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.9770879745483398,
|
||||
"mean": 0.1974783968180418,
|
||||
"median": 0.10099200159311295,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.2660581738745569
|
||||
},
|
||||
{
|
||||
"max": 0.892799973487854,
|
||||
"mean": 0.18164799660444259,
|
||||
"median": 0.1021759994328022,
|
||||
"min": 0.06435199826955795,
|
||||
"std": 0.23947520188197013
|
||||
},
|
||||
{
|
||||
"max": 0.6467199921607971,
|
||||
"mean": 0.15839359983801843,
|
||||
"median": 0.10100800171494484,
|
||||
"min": 0.06800000369548798,
|
||||
"std": 0.16617013141866102
|
||||
},
|
||||
{
|
||||
"max": 0.6725760102272034,
|
||||
"mean": 0.15686400160193442,
|
||||
"median": 0.10044800117611885,
|
||||
"min": 0.06063999980688095,
|
||||
"std": 0.17523222161300497
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12694399803876877,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.6659520268440247,
|
||||
"mean": 0.1927648030221462,
|
||||
"median": 0.12694399803876877,
|
||||
"min": 0.07609599828720093,
|
||||
"std": 0.16697784531907736
|
||||
},
|
||||
{
|
||||
"max": 0.695360004901886,
|
||||
"mean": 0.19356480240821838,
|
||||
"median": 0.11726400256156921,
|
||||
"min": 0.0796160027384758,
|
||||
"std": 0.17588096678867862
|
||||
},
|
||||
{
|
||||
"max": 0.5939840078353882,
|
||||
"mean": 0.1868800014257431,
|
||||
"median": 0.12379200011491776,
|
||||
"min": 0.07427199929952621,
|
||||
"std": 0.1463231714943902
|
||||
},
|
||||
{
|
||||
"max": 0.6635839939117432,
|
||||
"mean": 0.1874335989356041,
|
||||
"median": 0.12014400213956833,
|
||||
"min": 0.07526399940252304,
|
||||
"std": 0.16680959677760304
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09161599725484848,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12665599584579468,
|
||||
"mean": 0.09443839862942696,
|
||||
"median": 0.09124799817800522,
|
||||
"min": 0.06431999802589417,
|
||||
"std": 0.0205690155775906
|
||||
},
|
||||
{
|
||||
"max": 0.1303039938211441,
|
||||
"mean": 0.09712959825992584,
|
||||
"median": 0.09161599725484848,
|
||||
"min": 0.07648000121116638,
|
||||
"std": 0.019072048129173236
|
||||
},
|
||||
{
|
||||
"max": 0.13836799561977386,
|
||||
"mean": 0.09821119979023933,
|
||||
"median": 0.09148800000548363,
|
||||
"min": 0.0727040022611618,
|
||||
"std": 0.021314066545189116
|
||||
},
|
||||
{
|
||||
"max": 0.12992000579833984,
|
||||
"mean": 0.09272959977388381,
|
||||
"median": 0.08931199833750725,
|
||||
"min": 0.06406400352716446,
|
||||
"std": 0.021360803830354765
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08580800145864487,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13449600338935852,
|
||||
"mean": 0.08960640132427215,
|
||||
"median": 0.08299200236797333,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.020633597898445998
|
||||
},
|
||||
{
|
||||
"max": 0.14735999703407288,
|
||||
"mean": 0.09248319901525974,
|
||||
"median": 0.08580800145864487,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.025960234928829564
|
||||
},
|
||||
{
|
||||
"max": 0.13705599308013916,
|
||||
"mean": 0.08947199806571007,
|
||||
"median": 0.08460799977183342,
|
||||
"min": 0.0634239986538887,
|
||||
"std": 0.02122838946992339
|
||||
},
|
||||
{
|
||||
"max": 0.13846400380134583,
|
||||
"mean": 0.0859104000031948,
|
||||
"median": 0.08308799937367439,
|
||||
"min": 0.05974400043487549,
|
||||
"std": 0.022373631424433445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09867199882864952,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1279039978981018,
|
||||
"mean": 0.09748799875378608,
|
||||
"median": 0.09678399935364723,
|
||||
"min": 0.0655359998345375,
|
||||
"std": 0.02315719144614622
|
||||
},
|
||||
{
|
||||
"max": 0.1356479972600937,
|
||||
"mean": 0.10018239840865135,
|
||||
"median": 0.09532799944281578,
|
||||
"min": 0.06185600161552429,
|
||||
"std": 0.02330700253650042
|
||||
},
|
||||
{
|
||||
"max": 0.13142399489879608,
|
||||
"mean": 0.09778879955410957,
|
||||
"median": 0.09492799639701843,
|
||||
"min": 0.06560000032186508,
|
||||
"std": 0.02307579212430045
|
||||
},
|
||||
{
|
||||
"max": 0.1276479959487915,
|
||||
"mean": 0.09611519873142242,
|
||||
"median": 0.09867199882864952,
|
||||
"min": 0.0642239972949028,
|
||||
"std": 0.02273612181973431
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09646400064229965,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14060799777507782,
|
||||
"mean": 0.09770880043506622,
|
||||
"median": 0.09115200117230415,
|
||||
"min": 0.06950400024652481,
|
||||
"std": 0.024752645089849798
|
||||
},
|
||||
{
|
||||
"max": 0.14377599954605103,
|
||||
"mean": 0.09824960008263588,
|
||||
"median": 0.08999999985098839,
|
||||
"min": 0.07103999704122543,
|
||||
"std": 0.025403407389046027
|
||||
},
|
||||
{
|
||||
"max": 0.13680000603199005,
|
||||
"mean": 0.09993600100278854,
|
||||
"median": 0.09646400064229965,
|
||||
"min": 0.06790400296449661,
|
||||
"std": 0.022297424273985882
|
||||
},
|
||||
{
|
||||
"max": 0.1391039937734604,
|
||||
"mean": 0.09769919961690902,
|
||||
"median": 0.09601600095629692,
|
||||
"min": 0.06835199892520905,
|
||||
"std": 0.023899922100804445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08377600088715553,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1464959979057312,
|
||||
"mean": 0.09276160076260567,
|
||||
"median": 0.08377600088715553,
|
||||
"min": 0.06777600198984146,
|
||||
"std": 0.024751086465295658
|
||||
},
|
||||
{
|
||||
"max": 0.14319999516010284,
|
||||
"mean": 0.09080640003085136,
|
||||
"median": 0.080400001257658,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.023365718881708488
|
||||
},
|
||||
{
|
||||
"max": 0.1382399946451187,
|
||||
"mean": 0.09063360020518303,
|
||||
"median": 0.08193599805235863,
|
||||
"min": 0.06627199798822403,
|
||||
"std": 0.02372618650854302
|
||||
},
|
||||
{
|
||||
"max": 0.14313599467277527,
|
||||
"mean": 0.09044799953699112,
|
||||
"median": 0.08128000050783157,
|
||||
"min": 0.06652799993753433,
|
||||
"std": 0.02462486862033686
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1128000020980835,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1600320041179657,
|
||||
"mean": 0.11426239982247352,
|
||||
"median": 0.1128000020980835,
|
||||
"min": 0.07657600194215775,
|
||||
"std": 0.030240087702350687
|
||||
},
|
||||
{
|
||||
"max": 0.15881599485874176,
|
||||
"mean": 0.11206399947404862,
|
||||
"median": 0.10628800094127655,
|
||||
"min": 0.0772159993648529,
|
||||
"std": 0.029933135345483627
|
||||
},
|
||||
{
|
||||
"max": 0.15612800419330597,
|
||||
"mean": 0.10761600062251091,
|
||||
"median": 0.09860799834132195,
|
||||
"min": 0.07689599692821503,
|
||||
"std": 0.02634237020678647
|
||||
},
|
||||
{
|
||||
"max": 0.15865600109100342,
|
||||
"mean": 0.11094079986214637,
|
||||
"median": 0.10979199782013893,
|
||||
"min": 0.07583999633789062,
|
||||
"std": 0.028619217028542445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08755199983716011,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11999999731779099,
|
||||
"mean": 0.0843871995806694,
|
||||
"median": 0.08755199983716011,
|
||||
"min": 0.06332799792289734,
|
||||
"std": 0.016638543890716024
|
||||
},
|
||||
{
|
||||
"max": 0.1218239963054657,
|
||||
"mean": 0.08518079966306687,
|
||||
"median": 0.08032000064849854,
|
||||
"min": 0.06393600255250931,
|
||||
"std": 0.019900899429956945
|
||||
},
|
||||
{
|
||||
"max": 0.11849600076675415,
|
||||
"mean": 0.0843968003988266,
|
||||
"median": 0.08702399954199791,
|
||||
"min": 0.06297600269317627,
|
||||
"std": 0.017031908773433545
|
||||
},
|
||||
{
|
||||
"max": 0.12300799787044525,
|
||||
"mean": 0.0846304003149271,
|
||||
"median": 0.08139199763536453,
|
||||
"min": 0.06195199862122536,
|
||||
"std": 0.020106523698622265
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12361599877476692,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.17155200242996216,
|
||||
"mean": 0.12168959975242614,
|
||||
"median": 0.12055999785661697,
|
||||
"min": 0.09609600156545639,
|
||||
"std": 0.022565485532483928
|
||||
},
|
||||
{
|
||||
"max": 0.9246399998664856,
|
||||
"mean": 0.19978560134768486,
|
||||
"median": 0.12361599877476692,
|
||||
"min": 0.09715200215578079,
|
||||
"std": 0.24230694305662265
|
||||
},
|
||||
{
|
||||
"max": 0.9317439794540405,
|
||||
"mean": 0.20037759989500045,
|
||||
"median": 0.12327999994158745,
|
||||
"min": 0.09603200107812881,
|
||||
"std": 0.24450903278000383
|
||||
},
|
||||
{
|
||||
"max": 0.9321280121803284,
|
||||
"mean": 0.19875840097665787,
|
||||
"median": 0.12230399996042252,
|
||||
"min": 0.0950080007314682,
|
||||
"std": 0.24519252220785093
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.20030399411916733,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.3261440098285675,
|
||||
"mean": 0.20826880186796187,
|
||||
"median": 0.19974400103092194,
|
||||
"min": 0.1409280002117157,
|
||||
"std": 0.051397264855718945
|
||||
},
|
||||
{
|
||||
"max": 0.3248000144958496,
|
||||
"mean": 0.20548800230026246,
|
||||
"median": 0.1979840025305748,
|
||||
"min": 0.141184002161026,
|
||||
"std": 0.04980002399656717
|
||||
},
|
||||
{
|
||||
"max": 0.32547199726104736,
|
||||
"mean": 0.21280319690704347,
|
||||
"median": 0.20030399411916733,
|
||||
"min": 0.14127999544143677,
|
||||
"std": 0.05304243085685509
|
||||
},
|
||||
{
|
||||
"max": 0.26047998666763306,
|
||||
"mean": 0.1969312012195587,
|
||||
"median": 0.18079999834299088,
|
||||
"min": 0.14057600498199463,
|
||||
"std": 0.042174123638424224
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.2924960106611252,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.45052799582481384,
|
||||
"mean": 0.3013375997543335,
|
||||
"median": 0.28273600339889526,
|
||||
"min": 0.21334399282932281,
|
||||
"std": 0.07214223764527236
|
||||
},
|
||||
{
|
||||
"max": 0.4466240108013153,
|
||||
"mean": 0.2968191936612129,
|
||||
"median": 0.27796798944473267,
|
||||
"min": 0.21241599321365356,
|
||||
"std": 0.07400678240849741
|
||||
},
|
||||
{
|
||||
"max": 0.3830080032348633,
|
||||
"mean": 0.2946112036705017,
|
||||
"median": 0.2924960106611252,
|
||||
"min": 0.21084800362586975,
|
||||
"std": 0.05323627615746332
|
||||
},
|
||||
{
|
||||
"max": 0.4609600007534027,
|
||||
"mean": 0.3054272010922432,
|
||||
"median": 0.289792001247406,
|
||||
"min": 0.21062399446964264,
|
||||
"std": 0.07608482904865672
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
}
|
||||
],
|
||||
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
|
||||
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
|
@@ -0,0 +1,31 @@
|
||||
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,representation
|
||||
1,q64_8q1s512,1,8,64,8,511.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q128_8q1s1k,1,8,128,8,1023.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q128_16q1s1k,1,16,128,16,1023.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q256_16q1s2k,1,16,256,16,2047.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q256_32q1s2k,1,32,256,32,2047.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q512_32q1s4k,1,32,512,32,4095.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q512_64q1s4k,1,64,512,64,4095.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q1k_64q1s8k,1,64,1024,64,8191.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q2k_32q1s4k,1,32,2048,32,4095.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
1,q4k_16q1s4k,1,16,4096,16,4095.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q64_8q1s512,1,8,64,8,511.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q128_8q1s1k,1,8,128,8,1023.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q128_16q1s1k,1,16,128,16,1023.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q256_16q1s2k,1,16,256,16,2047.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q256_32q1s2k,1,32,256,32,2047.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q512_32q1s4k,1,32,512,32,4095.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q512_64q1s4k,1,64,512,64,4095.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q1k_64q1s8k,1,64,1024,64,8191.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q2k_32q1s4k,1,32,2048,32,4095.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
2,q4k_16q1s4k,1,16,4096,16,4095.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q64_8q1s512,1,8,64,8,511.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q128_8q1s1k,1,8,128,8,1023.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q128_16q1s1k,1,16,128,16,1023.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q256_16q1s2k,1,16,256,16,2047.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q256_32q1s2k,1,32,256,32,2047.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q512_32q1s4k,1,32,512,32,4095.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q512_64q1s4k,1,64,512,64,4095.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q1k_64q1s8k,1,64,1024,64,8191.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q2k_32q1s4k,1,32,2048,32,4095.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
4,q4k_16q1s4k,1,16,4096,16,4095.0,0.456512001156807,0.456512001156807,0.017680000513792038,one_fused_FA3_call_not_split_into_prefill_decode
|
||||
|
@@ -0,0 +1,37 @@
|
||||
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
|
||||
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
|
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "674e010598eea4c06bdee04ccba413f4c34194fbe2d70b143c6ec2169b638cc5",
|
||||
"attention_true_mixed_fused.csv": "92cd22f8b669abc7af488cf3d067ed880a6cced2dba1a3213a93a4f02a0001a8",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target",
|
||||
"attention_true_mixed": "Preserved as fused-total diagnostics and excluded from attention.csv because Frontier requires an unobservable prefill/decode split",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 102,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v1"
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
|
@@ -0,0 +1,824 @@
|
||||
{
|
||||
"environment": [
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
{
|
||||
"backend_env": {
|
||||
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
||||
},
|
||||
"gpu": "NVIDIA H20",
|
||||
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
||||
"torch_cuda": "12.9",
|
||||
"torch_version": "2.11.0+cu129",
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
}
|
||||
],
|
||||
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
|
||||
"rows": [
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08288000151515007,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.4872959852218628,
|
||||
"mean": 0.1310015980154276,
|
||||
"median": 0.07679999992251396,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.12790721677293843
|
||||
},
|
||||
{
|
||||
"max": 0.4402880072593689,
|
||||
"mean": 0.12842560112476348,
|
||||
"median": 0.08288000151515007,
|
||||
"min": 0.0655680000782013,
|
||||
"std": 0.11220176524616535
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0793600007891655,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12716799974441528,
|
||||
"mean": 0.07871360033750534,
|
||||
"median": 0.0759200006723404,
|
||||
"min": 0.06032000109553337,
|
||||
"std": 0.019331314939874535
|
||||
},
|
||||
{
|
||||
"max": 0.12380799651145935,
|
||||
"mean": 0.08059840016067028,
|
||||
"median": 0.0793600007891655,
|
||||
"min": 0.06217600032687187,
|
||||
"std": 0.01687088356254092
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0713919997215271,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12697599828243256,
|
||||
"mean": 0.0767391998320818,
|
||||
"median": 0.07078400254249573,
|
||||
"min": 0.05910399928689003,
|
||||
"std": 0.018775178979463278
|
||||
},
|
||||
{
|
||||
"max": 0.11430399864912033,
|
||||
"mean": 0.07594559974968433,
|
||||
"median": 0.0713919997215271,
|
||||
"min": 0.06124800071120262,
|
||||
"std": 0.015719922150631036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08056000247597694,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1037760004401207,
|
||||
"mean": 0.07954559996724128,
|
||||
"median": 0.08056000247597694,
|
||||
"min": 0.05955199897289276,
|
||||
"std": 0.0135697420393132
|
||||
},
|
||||
{
|
||||
"max": 0.10608000308275223,
|
||||
"mean": 0.07971520014107228,
|
||||
"median": 0.07593599706888199,
|
||||
"min": 0.06028800085186958,
|
||||
"std": 0.015319849772775456
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.0865279994904995,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14470399916172028,
|
||||
"mean": 0.09121599942445754,
|
||||
"median": 0.0865279994904995,
|
||||
"min": 0.06441599875688553,
|
||||
"std": 0.024893837894277456
|
||||
},
|
||||
{
|
||||
"max": 0.12438400089740753,
|
||||
"mean": 0.08531199917197227,
|
||||
"median": 0.08031999692320824,
|
||||
"min": 0.06364800035953522,
|
||||
"std": 0.01878029830059533
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07135999947786331,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11753600090742111,
|
||||
"mean": 0.07606079950928687,
|
||||
"median": 0.07135999947786331,
|
||||
"min": 0.05843200162053108,
|
||||
"std": 0.01755519771639284
|
||||
},
|
||||
{
|
||||
"max": 0.1103999987244606,
|
||||
"mean": 0.07607359997928143,
|
||||
"median": 0.07073600217700005,
|
||||
"min": 0.05721599981188774,
|
||||
"std": 0.016904445220949783
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.07321599870920181,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11740799993276596,
|
||||
"mean": 0.07749119997024537,
|
||||
"median": 0.07203200086951256,
|
||||
"min": 0.05862399935722351,
|
||||
"std": 0.017701381594822835
|
||||
},
|
||||
{
|
||||
"max": 0.11382400244474411,
|
||||
"mean": 0.07733759954571724,
|
||||
"median": 0.07321599870920181,
|
||||
"min": 0.059039998799562454,
|
||||
"std": 0.017188890882557036
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09025600180029869,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13980799913406372,
|
||||
"mean": 0.0950367994606495,
|
||||
"median": 0.09025600180029869,
|
||||
"min": 0.06815999746322632,
|
||||
"std": 0.022829835127539378
|
||||
},
|
||||
{
|
||||
"max": 0.14764800667762756,
|
||||
"mean": 0.09710080176591873,
|
||||
"median": 0.08720000088214874,
|
||||
"min": 0.07152000069618225,
|
||||
"std": 0.02500574954063789
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08083200082182884,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.15574400126934052,
|
||||
"mean": 0.08760640025138855,
|
||||
"median": 0.07846399769186974,
|
||||
"min": 0.07097599655389786,
|
||||
"std": 0.024487311423551025
|
||||
},
|
||||
{
|
||||
"max": 0.16284799575805664,
|
||||
"mean": 0.08963519930839539,
|
||||
"median": 0.08083200082182884,
|
||||
"min": 0.07100799679756165,
|
||||
"std": 0.026047320562445356
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.10891199856996536,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1582079976797104,
|
||||
"mean": 0.11541439890861512,
|
||||
"median": 0.10836799815297127,
|
||||
"min": 0.09062399715185165,
|
||||
"std": 0.018285136316576037
|
||||
},
|
||||
{
|
||||
"max": 0.1578879952430725,
|
||||
"mean": 0.11537599861621857,
|
||||
"median": 0.10891199856996536,
|
||||
"min": 0.0960640013217926,
|
||||
"std": 0.018246538626977286
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1703840047121048,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.19327999651432037,
|
||||
"mean": 0.1707327976822853,
|
||||
"median": 0.1703840047121048,
|
||||
"min": 0.14815999567508698,
|
||||
"std": 0.014211056022719618
|
||||
},
|
||||
{
|
||||
"max": 0.19276799261569977,
|
||||
"mean": 0.1658592015504837,
|
||||
"median": 0.16379200667142868,
|
||||
"min": 0.14560000598430634,
|
||||
"std": 0.013840249648693638
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.25539200007915497,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.2807359993457794,
|
||||
"mean": 0.25750079900026324,
|
||||
"median": 0.25539200007915497,
|
||||
"min": 0.24624000489711761,
|
||||
"std": 0.008925204570802302
|
||||
},
|
||||
{
|
||||
"max": 0.2863999903202057,
|
||||
"mean": 0.2585055992007256,
|
||||
"median": 0.255280002951622,
|
||||
"min": 0.24371199309825897,
|
||||
"std": 0.012059738582656496
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 2
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1021759994328022,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1,
|
||||
"payload_bytes": 4096,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.9770879745483398,
|
||||
"mean": 0.1974783968180418,
|
||||
"median": 0.10099200159311295,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.2660581738745569
|
||||
},
|
||||
{
|
||||
"max": 0.892799973487854,
|
||||
"mean": 0.18164799660444259,
|
||||
"median": 0.1021759994328022,
|
||||
"min": 0.06435199826955795,
|
||||
"std": 0.23947520188197013
|
||||
},
|
||||
{
|
||||
"max": 0.6467199921607971,
|
||||
"mean": 0.15839359983801843,
|
||||
"median": 0.10100800171494484,
|
||||
"min": 0.06800000369548798,
|
||||
"std": 0.16617013141866102
|
||||
},
|
||||
{
|
||||
"max": 0.6725760102272034,
|
||||
"mean": 0.15686400160193442,
|
||||
"median": 0.10044800117611885,
|
||||
"min": 0.06063999980688095,
|
||||
"std": 0.17523222161300497
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12694399803876877,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8,
|
||||
"payload_bytes": 32768,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.6659520268440247,
|
||||
"mean": 0.1927648030221462,
|
||||
"median": 0.12694399803876877,
|
||||
"min": 0.07609599828720093,
|
||||
"std": 0.16697784531907736
|
||||
},
|
||||
{
|
||||
"max": 0.695360004901886,
|
||||
"mean": 0.19356480240821838,
|
||||
"median": 0.11726400256156921,
|
||||
"min": 0.0796160027384758,
|
||||
"std": 0.17588096678867862
|
||||
},
|
||||
{
|
||||
"max": 0.5939840078353882,
|
||||
"mean": 0.1868800014257431,
|
||||
"median": 0.12379200011491776,
|
||||
"min": 0.07427199929952621,
|
||||
"std": 0.1463231714943902
|
||||
},
|
||||
{
|
||||
"max": 0.6635839939117432,
|
||||
"mean": 0.1874335989356041,
|
||||
"median": 0.12014400213956833,
|
||||
"min": 0.07526399940252304,
|
||||
"std": 0.16680959677760304
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09161599725484848,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 16,
|
||||
"payload_bytes": 65536,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.12665599584579468,
|
||||
"mean": 0.09443839862942696,
|
||||
"median": 0.09124799817800522,
|
||||
"min": 0.06431999802589417,
|
||||
"std": 0.0205690155775906
|
||||
},
|
||||
{
|
||||
"max": 0.1303039938211441,
|
||||
"mean": 0.09712959825992584,
|
||||
"median": 0.09161599725484848,
|
||||
"min": 0.07648000121116638,
|
||||
"std": 0.019072048129173236
|
||||
},
|
||||
{
|
||||
"max": 0.13836799561977386,
|
||||
"mean": 0.09821119979023933,
|
||||
"median": 0.09148800000548363,
|
||||
"min": 0.0727040022611618,
|
||||
"std": 0.021314066545189116
|
||||
},
|
||||
{
|
||||
"max": 0.12992000579833984,
|
||||
"mean": 0.09272959977388381,
|
||||
"median": 0.08931199833750725,
|
||||
"min": 0.06406400352716446,
|
||||
"std": 0.021360803830354765
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08580800145864487,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 32,
|
||||
"payload_bytes": 131072,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.13449600338935852,
|
||||
"mean": 0.08960640132427215,
|
||||
"median": 0.08299200236797333,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.020633597898445998
|
||||
},
|
||||
{
|
||||
"max": 0.14735999703407288,
|
||||
"mean": 0.09248319901525974,
|
||||
"median": 0.08580800145864487,
|
||||
"min": 0.05913599953055382,
|
||||
"std": 0.025960234928829564
|
||||
},
|
||||
{
|
||||
"max": 0.13705599308013916,
|
||||
"mean": 0.08947199806571007,
|
||||
"median": 0.08460799977183342,
|
||||
"min": 0.0634239986538887,
|
||||
"std": 0.02122838946992339
|
||||
},
|
||||
{
|
||||
"max": 0.13846400380134583,
|
||||
"mean": 0.0859104000031948,
|
||||
"median": 0.08308799937367439,
|
||||
"min": 0.05974400043487549,
|
||||
"std": 0.022373631424433445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09867199882864952,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 64,
|
||||
"payload_bytes": 262144,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1279039978981018,
|
||||
"mean": 0.09748799875378608,
|
||||
"median": 0.09678399935364723,
|
||||
"min": 0.0655359998345375,
|
||||
"std": 0.02315719144614622
|
||||
},
|
||||
{
|
||||
"max": 0.1356479972600937,
|
||||
"mean": 0.10018239840865135,
|
||||
"median": 0.09532799944281578,
|
||||
"min": 0.06185600161552429,
|
||||
"std": 0.02330700253650042
|
||||
},
|
||||
{
|
||||
"max": 0.13142399489879608,
|
||||
"mean": 0.09778879955410957,
|
||||
"median": 0.09492799639701843,
|
||||
"min": 0.06560000032186508,
|
||||
"std": 0.02307579212430045
|
||||
},
|
||||
{
|
||||
"max": 0.1276479959487915,
|
||||
"mean": 0.09611519873142242,
|
||||
"median": 0.09867199882864952,
|
||||
"min": 0.0642239972949028,
|
||||
"std": 0.02273612181973431
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.09646400064229965,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 128,
|
||||
"payload_bytes": 524288,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.14060799777507782,
|
||||
"mean": 0.09770880043506622,
|
||||
"median": 0.09115200117230415,
|
||||
"min": 0.06950400024652481,
|
||||
"std": 0.024752645089849798
|
||||
},
|
||||
{
|
||||
"max": 0.14377599954605103,
|
||||
"mean": 0.09824960008263588,
|
||||
"median": 0.08999999985098839,
|
||||
"min": 0.07103999704122543,
|
||||
"std": 0.025403407389046027
|
||||
},
|
||||
{
|
||||
"max": 0.13680000603199005,
|
||||
"mean": 0.09993600100278854,
|
||||
"median": 0.09646400064229965,
|
||||
"min": 0.06790400296449661,
|
||||
"std": 0.022297424273985882
|
||||
},
|
||||
{
|
||||
"max": 0.1391039937734604,
|
||||
"mean": 0.09769919961690902,
|
||||
"median": 0.09601600095629692,
|
||||
"min": 0.06835199892520905,
|
||||
"std": 0.023899922100804445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08377600088715553,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 256,
|
||||
"payload_bytes": 1048576,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1464959979057312,
|
||||
"mean": 0.09276160076260567,
|
||||
"median": 0.08377600088715553,
|
||||
"min": 0.06777600198984146,
|
||||
"std": 0.024751086465295658
|
||||
},
|
||||
{
|
||||
"max": 0.14319999516010284,
|
||||
"mean": 0.09080640003085136,
|
||||
"median": 0.080400001257658,
|
||||
"min": 0.06796800345182419,
|
||||
"std": 0.023365718881708488
|
||||
},
|
||||
{
|
||||
"max": 0.1382399946451187,
|
||||
"mean": 0.09063360020518303,
|
||||
"median": 0.08193599805235863,
|
||||
"min": 0.06627199798822403,
|
||||
"std": 0.02372618650854302
|
||||
},
|
||||
{
|
||||
"max": 0.14313599467277527,
|
||||
"mean": 0.09044799953699112,
|
||||
"median": 0.08128000050783157,
|
||||
"min": 0.06652799993753433,
|
||||
"std": 0.02462486862033686
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.1128000020980835,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 512,
|
||||
"payload_bytes": 2097152,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.1600320041179657,
|
||||
"mean": 0.11426239982247352,
|
||||
"median": 0.1128000020980835,
|
||||
"min": 0.07657600194215775,
|
||||
"std": 0.030240087702350687
|
||||
},
|
||||
{
|
||||
"max": 0.15881599485874176,
|
||||
"mean": 0.11206399947404862,
|
||||
"median": 0.10628800094127655,
|
||||
"min": 0.0772159993648529,
|
||||
"std": 0.029933135345483627
|
||||
},
|
||||
{
|
||||
"max": 0.15612800419330597,
|
||||
"mean": 0.10761600062251091,
|
||||
"median": 0.09860799834132195,
|
||||
"min": 0.07689599692821503,
|
||||
"std": 0.02634237020678647
|
||||
},
|
||||
{
|
||||
"max": 0.15865600109100342,
|
||||
"mean": 0.11094079986214637,
|
||||
"median": 0.10979199782013893,
|
||||
"min": 0.07583999633789062,
|
||||
"std": 0.028619217028542445
|
||||
}
|
||||
],
|
||||
"selected_backend": "flashinfer_trtllm",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.08755199983716011,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 1024,
|
||||
"payload_bytes": 4194304,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.11999999731779099,
|
||||
"mean": 0.0843871995806694,
|
||||
"median": 0.08755199983716011,
|
||||
"min": 0.06332799792289734,
|
||||
"std": 0.016638543890716024
|
||||
},
|
||||
{
|
||||
"max": 0.1218239963054657,
|
||||
"mean": 0.08518079966306687,
|
||||
"median": 0.08032000064849854,
|
||||
"min": 0.06393600255250931,
|
||||
"std": 0.019900899429956945
|
||||
},
|
||||
{
|
||||
"max": 0.11849600076675415,
|
||||
"mean": 0.0843968003988266,
|
||||
"median": 0.08702399954199791,
|
||||
"min": 0.06297600269317627,
|
||||
"std": 0.017031908773433545
|
||||
},
|
||||
{
|
||||
"max": 0.12300799787044525,
|
||||
"mean": 0.0846304003149271,
|
||||
"median": 0.08139199763536453,
|
||||
"min": 0.06195199862122536,
|
||||
"std": 0.020106523698622265
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.12361599877476692,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 2048,
|
||||
"payload_bytes": 8388608,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.17155200242996216,
|
||||
"mean": 0.12168959975242614,
|
||||
"median": 0.12055999785661697,
|
||||
"min": 0.09609600156545639,
|
||||
"std": 0.022565485532483928
|
||||
},
|
||||
{
|
||||
"max": 0.9246399998664856,
|
||||
"mean": 0.19978560134768486,
|
||||
"median": 0.12361599877476692,
|
||||
"min": 0.09715200215578079,
|
||||
"std": 0.24230694305662265
|
||||
},
|
||||
{
|
||||
"max": 0.9317439794540405,
|
||||
"mean": 0.20037759989500045,
|
||||
"median": 0.12327999994158745,
|
||||
"min": 0.09603200107812881,
|
||||
"std": 0.24450903278000383
|
||||
},
|
||||
{
|
||||
"max": 0.9321280121803284,
|
||||
"mean": 0.19875840097665787,
|
||||
"median": 0.12230399996042252,
|
||||
"min": 0.0950080007314682,
|
||||
"std": 0.24519252220785093
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.20030399411916733,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 4096,
|
||||
"payload_bytes": 16777216,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.3261440098285675,
|
||||
"mean": 0.20826880186796187,
|
||||
"median": 0.19974400103092194,
|
||||
"min": 0.1409280002117157,
|
||||
"std": 0.051397264855718945
|
||||
},
|
||||
{
|
||||
"max": 0.3248000144958496,
|
||||
"mean": 0.20548800230026246,
|
||||
"median": 0.1979840025305748,
|
||||
"min": 0.141184002161026,
|
||||
"std": 0.04980002399656717
|
||||
},
|
||||
{
|
||||
"max": 0.32547199726104736,
|
||||
"mean": 0.21280319690704347,
|
||||
"median": 0.20030399411916733,
|
||||
"min": 0.14127999544143677,
|
||||
"std": 0.05304243085685509
|
||||
},
|
||||
{
|
||||
"max": 0.26047998666763306,
|
||||
"mean": 0.1969312012195587,
|
||||
"median": 0.18079999834299088,
|
||||
"min": 0.14057600498199463,
|
||||
"std": 0.042174123638424224
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
},
|
||||
{
|
||||
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
||||
"critical_path_median_ms": 0.2924960106611252,
|
||||
"dtype": "bfloat16",
|
||||
"hidden_dim": 2048,
|
||||
"num_tokens": 8192,
|
||||
"payload_bytes": 33554432,
|
||||
"per_rank_time_ms": [
|
||||
{
|
||||
"max": 0.45052799582481384,
|
||||
"mean": 0.3013375997543335,
|
||||
"median": 0.28273600339889526,
|
||||
"min": 0.21334399282932281,
|
||||
"std": 0.07214223764527236
|
||||
},
|
||||
{
|
||||
"max": 0.4466240108013153,
|
||||
"mean": 0.2968191936612129,
|
||||
"median": 0.27796798944473267,
|
||||
"min": 0.21241599321365356,
|
||||
"std": 0.07400678240849741
|
||||
},
|
||||
{
|
||||
"max": 0.3830080032348633,
|
||||
"mean": 0.2946112036705017,
|
||||
"median": 0.2924960106611252,
|
||||
"min": 0.21084800362586975,
|
||||
"std": 0.05323627615746332
|
||||
},
|
||||
{
|
||||
"max": 0.4609600007534027,
|
||||
"mean": 0.3054272010922432,
|
||||
"median": 0.289792001247406,
|
||||
"min": 0.21062399446964264,
|
||||
"std": 0.07608482904865672
|
||||
}
|
||||
],
|
||||
"selected_backend": "nccl_fallback",
|
||||
"tensor_parallel_size": 4
|
||||
}
|
||||
],
|
||||
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
|
||||
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
|
||||
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
|
||||
|
@@ -0,0 +1,31 @@
|
||||
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation
|
||||
1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
|
||||
|
@@ -0,0 +1,37 @@
|
||||
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
|
||||
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
|
||||
|
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"attention_tp_coverage": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"environment_contract": {
|
||||
"dtype": "bfloat16",
|
||||
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
||||
"hardware": "NVIDIA H20",
|
||||
"model": "Qwen3-30B-A3B",
|
||||
"tensor_parallel_sizes": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
||||
"vllm_version": "0.20.0"
|
||||
},
|
||||
"inputs": {
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
|
||||
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
|
||||
},
|
||||
"outputs": {
|
||||
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
|
||||
"attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d",
|
||||
"attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60",
|
||||
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
|
||||
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
|
||||
},
|
||||
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving",
|
||||
"projection_contract": {
|
||||
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
|
||||
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
|
||||
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
|
||||
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
|
||||
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
|
||||
},
|
||||
"row_counts": {
|
||||
"allreduce": 24,
|
||||
"attention_frontier_compatible": 132,
|
||||
"attention_true_mixed_fused_diagnostic": 30,
|
||||
"linear": 36,
|
||||
"moe": 72
|
||||
},
|
||||
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v2"
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
|
||||
|
142
runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py
Normal file
142
runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py
Normal file
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the profile ablation and execution-context diagnostics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--s2", type=Path, required=True)
|
||||
parser.add_argument("--routing", type=Path, required=True)
|
||||
parser.add_argument("--opprof", type=Path, required=True)
|
||||
parser.add_argument("--p1", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
s2 = json.loads(args.s2.read_text())
|
||||
routing = json.loads(args.routing.read_text())
|
||||
opprof = json.loads(args.opprof.read_text())
|
||||
p1 = json.loads(args.p1.read_text())
|
||||
|
||||
plt.rcParams.update({"font.size": 9, "axes.titlesize": 10, "axes.labelsize": 9})
|
||||
fig, axes = plt.subplots(2, 2, figsize=(13.2, 8.2), constrained_layout=True)
|
||||
|
||||
real = s2["real_scores"]
|
||||
cells = sorted(real, key=lambda cell: (-real[cell], cell))
|
||||
x = np.arange(len(cells))
|
||||
series = (
|
||||
("Real", real, "#111111", "o"),
|
||||
(
|
||||
"Old profile-only",
|
||||
s2["historical_modes"]["historical-profile-only"]["simulated_scores"],
|
||||
"#d95f02",
|
||||
"s",
|
||||
),
|
||||
(
|
||||
"vLLM 0.20 profile-only",
|
||||
s2["vllm020_profile_only"]["SLO-gated"]["simulated_scores"],
|
||||
"#7570b3",
|
||||
"^",
|
||||
),
|
||||
(
|
||||
"Frozen per-TP calibration",
|
||||
s2["historical_modes"]["historical-per-tp-calibration"]["simulated_scores"],
|
||||
"#1b9e77",
|
||||
"D",
|
||||
),
|
||||
)
|
||||
ax = axes[0, 0]
|
||||
for label, values, color, marker in series:
|
||||
ax.plot(x, [values[cell] for cell in cells], label=label, color=color, marker=marker, lw=1.7)
|
||||
ax.set_xticks(x, [cell.replace("_", "\n") for cell in cells])
|
||||
ax.set_ylabel("SLO-feasible req/s/GPU")
|
||||
ax.set_title("(a) Full 92-probe config ranking")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
ax.legend(fontsize=8, ncol=2, loc="upper right")
|
||||
|
||||
ax = axes[0, 1]
|
||||
categories = ("Actual prefill", "Actual decode", "Frontier prior")
|
||||
cv = [
|
||||
routing["phase_summary"]["prefill"]["actual"]["load_cv"]["median"],
|
||||
routing["phase_summary"]["decode"]["actual"]["load_cv"]["median"],
|
||||
routing["phase_summary"]["prefill"]["frontier_simulation"]["load_cv"]["median"],
|
||||
]
|
||||
max_ratio = [
|
||||
routing["phase_summary"]["prefill"]["actual"]["max_load_ratio"]["median"],
|
||||
routing["phase_summary"]["decode"]["actual"]["max_load_ratio"]["median"],
|
||||
routing["phase_summary"]["prefill"]["frontier_simulation"]["max_load_ratio"]["median"],
|
||||
]
|
||||
bx = np.arange(len(categories))
|
||||
width = 0.34
|
||||
bars1 = ax.bar(bx - width / 2, cv, width, label="Load CV", color="#66c2a5")
|
||||
bars2 = ax.bar(bx + width / 2, max_ratio, width, label="Max/mean load", color="#fc8d62")
|
||||
ax.bar_label(bars1, fmt="%.2f", fontsize=8)
|
||||
ax.bar_label(bars2, fmt="%.2f", fontsize=8)
|
||||
ax.set_xticks(bx, categories)
|
||||
ax.set_ylim(0, max(max_ratio) * 1.2)
|
||||
ax.set_title("(b) Per-layer MoE routing skew")
|
||||
ax.legend(fontsize=8)
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
|
||||
graph_counts = {phase: Counter() for phase in ("pure_decode", "pure_prefill", "true_mixed")}
|
||||
for cell in opprof["cells"]:
|
||||
for group in cell["groups"]:
|
||||
graph_counts[group["phase"]][group["cudagraph_runtime_mode"]] += int(group["steps"])
|
||||
ax = axes[1, 0]
|
||||
phases = tuple(graph_counts)
|
||||
bottoms = np.zeros(len(phases))
|
||||
for mode, color in (("FULL", "#1b9e77"), ("PIECEWISE", "#e6ab02"), ("NONE", "#d95f02")):
|
||||
values = np.asarray(
|
||||
[100 * graph_counts[phase][mode] / sum(graph_counts[phase].values()) for phase in phases]
|
||||
)
|
||||
ax.bar(np.arange(len(phases)), values, bottom=bottoms, label=mode, color=color)
|
||||
bottoms += values
|
||||
ax.set_xticks(np.arange(len(phases)), [phase.replace("_", " ") for phase in phases])
|
||||
ax.set_ylabel("Observed scheduler steps (%)")
|
||||
ax.set_ylim(0, 100)
|
||||
ax.set_title("(c) Real vLLM execution mode is phase-dependent")
|
||||
ax.legend(fontsize=8, ncol=3, loc="lower left")
|
||||
|
||||
ax = axes[1, 1]
|
||||
mode_order = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only")
|
||||
labels = ("Per-TP\ncalibration", "Old\nprofile-only", "vLLM 0.20\nprofile-only")
|
||||
accuracy = [100 * p1["summaries"][mode]["probe_classification"]["accuracy"] for mode in mode_order]
|
||||
mae = [100 * p1["summaries"][mode]["pass_rate_mae"] for mode in mode_order]
|
||||
px = np.arange(len(labels))
|
||||
bars1 = ax.bar(px - width / 2, accuracy, width, label="Label accuracy (higher better)", color="#1b9e77")
|
||||
bars2 = ax.bar(px + width / 2, mae, width, label="Pass-rate MAE (lower better)", color="#d95f02")
|
||||
ax.bar_label(bars1, fmt="%.1f", fontsize=8)
|
||||
ax.bar_label(bars2, fmt="%.1f", fontsize=8)
|
||||
ax.set_xticks(px, labels)
|
||||
ax.set_ylabel("Percent")
|
||||
ax.set_ylim(0, 105)
|
||||
ax.set_title("(d) Held-out P1 boundary probes (12 labels)")
|
||||
ax.legend(fontsize=8, loc="upper right")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
|
||||
fig.suptitle(
|
||||
"Qwen3-30B-A3B / vLLM 0.20 / BF16 / dash0 H20: operator provenance is not execution-context fidelity",
|
||||
fontsize=12,
|
||||
)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(args.output, dpi=180)
|
||||
fig.savefig(args.output.with_suffix(".svg"))
|
||||
print(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare the 92 frozen S2-R-b probes with a replacement profile bundle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROFILE_KEYS = {
|
||||
"linear_op_input_file": "linear_op.csv",
|
||||
"atten_input_file": "attention.csv",
|
||||
"moe_input_file": "moe.csv",
|
||||
}
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--resolved-plan", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--profile-root", type=Path, required=True)
|
||||
parser.add_argument("--cache-root", type=Path)
|
||||
parser.add_argument("--shards", type=int, default=1)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.shards < 1:
|
||||
raise SystemExit("--shards must be positive")
|
||||
source = json.loads(args.resolved_plan.read_text())
|
||||
source_rows = [row for row in source["runs"] if row["mode"] == "uncalibrated"]
|
||||
if len(source_rows) != 92:
|
||||
raise SystemExit(f"expected 92 uncalibrated probes, found {len(source_rows)}")
|
||||
|
||||
output = args.output.resolve()
|
||||
profile_root = args.profile_root.resolve()
|
||||
cache_root = (args.cache_root or (output / "prediction-cache")).resolve()
|
||||
profile_hashes: dict[str, str] = {}
|
||||
for filename in PROFILE_KEYS.values():
|
||||
path = profile_root / filename
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing frozen profile: {path}")
|
||||
profile_hashes[str(path)] = sha256(path)
|
||||
|
||||
entries: list[dict[str, Any]] = []
|
||||
for row in source_rows:
|
||||
fixture_dir = Path(row["fixture_dir"]).resolve()
|
||||
fixture_manifest_path = fixture_dir / "fixture_manifest.json"
|
||||
fixture = json.loads(fixture_manifest_path.read_text())
|
||||
config = json.loads(Path(row["config_path"]).read_text())
|
||||
config["mode"] = "new-profile-only"
|
||||
config["config_id"] = f"{row['cell_id']}__new-profile-only"
|
||||
config["calibration"]["a_tp"] = 1.0
|
||||
knobs = config["frontier"]["knobs"]
|
||||
knobs["cache_dir"] = str(cache_root)
|
||||
knobs["no_cache"] = False
|
||||
for key, filename in PROFILE_KEYS.items():
|
||||
knobs[key] = str(profile_root / filename)
|
||||
|
||||
target_config = output / "configs" / f"{row['fixture_id']}.json"
|
||||
write_json(target_config, config)
|
||||
entries.append(
|
||||
{
|
||||
"fixture_id": row["fixture_id"],
|
||||
"cell": row["cell_id"],
|
||||
"role": f"probe-{int(row['probe_index']):02d}",
|
||||
"anchor": float(row["sampling_u"]),
|
||||
"selected_count": int(row["request_count"]),
|
||||
"probe_index": int(row["probe_index"]),
|
||||
"sampling_u": float(row["sampling_u"]),
|
||||
"tensor_parallel_size": int(row["tensor_parallel_size"]),
|
||||
"config": str(target_config),
|
||||
"fixture_manifest": str(fixture_manifest_path),
|
||||
"frontier_csv": str(fixture_dir / "frontier.csv"),
|
||||
"sidecar": str(fixture_dir / "sidecar.jsonl"),
|
||||
"calibration_scale": 1.0,
|
||||
"request_count": int(fixture["request_count"]),
|
||||
}
|
||||
)
|
||||
|
||||
base = {
|
||||
"schema": "frontier-qwen30-s2-profile-comparison-prepared.v1",
|
||||
"status": "PASS",
|
||||
"mode": "new-profile-only",
|
||||
"source": {
|
||||
"resolved_plan": str(args.resolved_plan.resolve()),
|
||||
"sha256": sha256(args.resolved_plan),
|
||||
},
|
||||
"profile_hashes": profile_hashes,
|
||||
"isolation": {
|
||||
"calibration_a_tp": 1.0,
|
||||
"prediction_cache": str(cache_root),
|
||||
"all_non_profile_knobs_inherited": True,
|
||||
},
|
||||
"suite_total_runs": len(entries),
|
||||
}
|
||||
write_json(
|
||||
output / "prepared-manifest.json",
|
||||
{**base, "expected_runs": len(entries), "entries": entries},
|
||||
)
|
||||
for shard in range(args.shards):
|
||||
shard_entries = entries[shard :: args.shards]
|
||||
write_json(
|
||||
output / f"prepared-manifest-shard-{shard:02d}-of-{args.shards:02d}.json",
|
||||
{
|
||||
**base,
|
||||
"shard": {"index": shard, "count": args.shards},
|
||||
"expected_runs": len(shard_entries),
|
||||
"entries": shard_entries,
|
||||
},
|
||||
)
|
||||
print(output / "prepared-manifest.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user