120 lines
4.2 KiB
Python
120 lines
4.2 KiB
Python
"""Generate benchmark summary for GitHub Actions."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from results import BenchmarkResult, GPUUtilization
|
|
|
|
|
|
def discover_benchmarks(base_dir: Path) -> list[tuple[Path, str]]:
|
|
"""Auto-discover benchmark folders and their result JSON files.
|
|
|
|
Returns list of (json_path, label) tuples sorted by folder name.
|
|
"""
|
|
results = []
|
|
for folder in base_dir.rglob("benchmark_*"):
|
|
if not folder.is_dir():
|
|
continue
|
|
# Find result JSON (exclude metadata and gpu files)
|
|
for json_file in folder.glob("*.json"):
|
|
if (
|
|
"experiment_metadata" not in json_file.name
|
|
and "gpu_utilization" not in json_file.name
|
|
):
|
|
# Generate label from folder name: benchmark_cache_aware_pd_grpc -> cache_aware pd grpc
|
|
label = folder.name.replace("benchmark_", "").replace("_", " ")
|
|
results.append((json_file, label))
|
|
break # One JSON per folder
|
|
return sorted(results, key=lambda x: x[0].parent.name)
|
|
|
|
|
|
def find_gpu_utilization(result_path: Path) -> Path | None:
|
|
"""Find GPU utilization JSON in same folder as result."""
|
|
gpu_json = result_path.parent / "gpu_utilization.json"
|
|
return gpu_json if gpu_json.exists() else None
|
|
|
|
|
|
def generate_summary(base_dir: Path) -> str:
|
|
"""Generate markdown summary."""
|
|
benchmarks = discover_benchmarks(base_dir)
|
|
|
|
if not benchmarks:
|
|
return (
|
|
"## Gateway E2E Genai-Bench Results Summary\n\nNo benchmark results found."
|
|
)
|
|
|
|
lines = [
|
|
"## Gateway E2E Genai-Bench Results Summary",
|
|
"",
|
|
"| Scenario | Status | TTFT (s) | E2E Latency (s) | Input Throughput (tok/s) | Output Throughput (tok/s) |",
|
|
"|----------|--------|----------|-----------------|--------------------------|---------------------------|",
|
|
]
|
|
|
|
gpu_sections = []
|
|
|
|
for result_path, label in benchmarks:
|
|
try:
|
|
result = BenchmarkResult.from_json(result_path)
|
|
except Exception as e:
|
|
print(f"Warning: Failed to parse {result_path}: {e}", file=sys.stderr)
|
|
lines.append(f"| {label} | ❌ Failed | - | - | - | - |")
|
|
continue
|
|
|
|
lines.append(
|
|
f"| {label} | ✅ Success | "
|
|
f"{result.ttft_mean:.2f} | "
|
|
f"{result.e2e_latency_mean:.2f} | "
|
|
f"{result.input_throughput_mean:.0f} | "
|
|
f"{result.output_throughput_mean:.0f} |"
|
|
)
|
|
|
|
# GPU utilization
|
|
gpu_path = find_gpu_utilization(result_path)
|
|
if gpu_path:
|
|
gpu = GPUUtilization.from_json(gpu_path)
|
|
if gpu and gpu.per_gpu:
|
|
gpu_lines = [
|
|
f"### GPU Utilization — {label}",
|
|
"",
|
|
f"Overall mean: {gpu.overall_mean:.2f}%",
|
|
"",
|
|
"| GPU | Mean (%) | p5 | p10 | p25 | p50 | p75 | p90 | p95 |",
|
|
"|-----|----------|----|-----|-----|-----|-----|-----|-----|",
|
|
]
|
|
for gpu_id, stats in sorted(
|
|
gpu.per_gpu.items(), key=lambda x: int(x[0])
|
|
):
|
|
gpu_lines.append(
|
|
f"| {gpu_id} | {stats.get('mean', 0):.2f} | "
|
|
f"{stats.get('p5', 0):.2f} | {stats.get('p10', 0):.2f} | "
|
|
f"{stats.get('p25', 0):.2f} | {stats.get('p50', 0):.2f} | "
|
|
f"{stats.get('p75', 0):.2f} | {stats.get('p90', 0):.2f} | "
|
|
f"{stats.get('p95', 0):.2f} |"
|
|
)
|
|
gpu_sections.append("\n".join(gpu_lines))
|
|
|
|
return "\n".join(lines) + "\n" + "\n\n".join(gpu_sections)
|
|
|
|
|
|
def main() -> None:
|
|
"""Main entry point."""
|
|
base_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path.cwd()
|
|
summary = generate_summary(base_dir)
|
|
|
|
# Write to GITHUB_STEP_SUMMARY if available
|
|
summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
if summary_file:
|
|
with open(summary_file, "a") as f:
|
|
f.write(summary)
|
|
f.write("\n")
|
|
print(f"Summary written to {summary_file}")
|
|
else:
|
|
print(summary)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|