chore: vendor sglang v0.5.10 snapshot

third_party/sglang/scripts/ci/utils/ci_coverage_report.py

@@ -0,0 +1,477 @@
+#!/usr/bin/env python3
+"""
+CI Coverage Report Generator
+
+Collects all CI test registrations from test/registered/ and generates
+a coverage report organized by folder, backend, and suite.
+
+Usage:
+    python scripts/ci/utils/ci_coverage_report.py [--output-format markdown|json]
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+# Add the ci_register module path directly to avoid heavy sglang imports
+sys.path.insert(
+    0,
+    str(
+        Path(__file__).parent.parent.parent.parent / "python" / "sglang" / "test" / "ci"
+    ),
+)
+
+from ci_register import CIRegistry, HWBackend, ut_parse_one_file
+
+
+def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
+    """Collect all CI registrations from registered directory."""
+    files = glob.glob(f"{registered_dir}/**/*.py", recursive=True)
+    all_tests = []
+
+    for file in sorted(files):
+        try:
+            registries = ut_parse_one_file(file)
+            all_tests.extend(registries)
+        except Exception as e:
+            print(f"Warning: Failed to parse {file}: {e}", file=sys.stderr)
+
+    return all_tests
+
+
+def get_folder_name(filename: str) -> str:
+    """Extract folder name from test filename."""
+    # e.g., "registered/models/test_foo.py" -> "models"
+    parts = Path(filename).parts
+    if "registered" in parts:
+        idx = parts.index("registered")
+        if idx + 1 < len(parts) - 1:  # Has subfolder
+            return parts[idx + 1]
+    return "root"
+
+
+def get_test_basename(filename: str) -> str:
+    """Extract just the test file name from the path."""
+    return Path(filename).name
+
+
+def organize_test_data(tests: list[CIRegistry]) -> dict:
+    """Organize tests into various groupings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+    disabled_tests = []
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+        if t.disabled:
+            disabled_tests.append(t)
+
+    # Count unique test files (a file may be registered for multiple backends)
+    unique_files = set(t.filename for t in tests)
+    unique_enabled_files = set(t.filename for t in tests if not t.disabled)
+    unique_disabled_files = set(t.filename for t in tests if t.disabled)
+
+    return {
+        "total": len(tests),
+        "total_unique_files": len(unique_files),
+        "enabled": len(tests) - len(disabled_tests),
+        "enabled_unique_files": len(unique_enabled_files),
+        "disabled_count": len(disabled_tests),
+        "disabled_unique_files": len(unique_disabled_files),
+        "by_backend": by_backend,
+        "by_folder": by_folder,
+        "disabled_tests": disabled_tests,
+    }
+
+
+def generate_summary_section(data: dict) -> str:
+    """Generate the summary/overview section."""
+    lines = []
+    lines.append("# CI Coverage Overview\n")
+    lines.append(
+        f"**Unique Test Files:** {data['total_unique_files']} ({data['enabled_unique_files']} enabled, {data['disabled_unique_files']} disabled)\n"
+    )
+    lines.append(
+        f"**Total Registrations:** {data['total']} ({data['enabled']} enabled, {data['disabled_count']} disabled)\n"
+    )
+    lines.append(
+        "*Note: A test file may be registered for multiple backends (e.g., CUDA + AMD), so total registrations > unique files.*\n"
+    )
+
+    by_backend = data["by_backend"]
+    by_folder = data["by_folder"]
+    disabled_tests = data["disabled_tests"]
+
+    # Backend summary (collapsible)
+    lines.append("<details>")
+    lines.append("<summary><h2>Backend Summary</h2></summary>\n")
+    lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |")
+    lines.append("|---------|-------|---------|----------|------------|---------|")
+
+    for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+        b_per_commit = sum(1 for t in backend_tests if not t.nightly and not t.disabled)
+        b_nightly = sum(1 for t in backend_tests if t.nightly and not t.disabled)
+        lines.append(
+            f"| {backend} | {b_total} | {b_enabled} | {b_disabled} | {b_per_commit} | {b_nightly} |"
+        )
+
+    lines.append("\n</details>\n")
+
+    # Folder summary (collapsible)
+    lines.append("<details>")
+    lines.append("<summary><h2>Folder Summary</h2></summary>\n")
+    lines.append("| Folder | CUDA | AMD | NPU | CPU | Total |")
+    lines.append("|--------|------|-----|-----|-----|-------|")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        cuda = sum(1 for t in folder_tests if t.backend == HWBackend.CUDA)
+        amd = sum(1 for t in folder_tests if t.backend == HWBackend.AMD)
+        npu = sum(1 for t in folder_tests if t.backend == HWBackend.NPU)
+        cpu = sum(1 for t in folder_tests if t.backend == HWBackend.CPU)
+        lines.append(
+            f"| {folder} | {cuda} | {amd} | {npu} | {cpu} | {len(folder_tests)} |"
+        )
+
+    lines.append("\n</details>\n")
+
+    # Disabled tests section (collapsible)
+    if disabled_tests:
+        lines.append("<details>")
+        lines.append("<summary><h2>Disabled Tests</h2></summary>\n")
+        lines.append("| File | Backend | Suite | Reason |")
+        lines.append("|------|---------|-------|--------|")
+        for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+            test_name = get_test_basename(t.filename)
+            reason = t.disabled[:50] + "..." if len(t.disabled) > 50 else t.disabled
+            lines.append(f"| `{test_name}` | {t.backend.name} | {t.suite} | {reason} |")
+        lines.append("\n</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_folder_section(data: dict) -> str:
+    """Generate the 'All Tests by Folder' section."""
+    lines = []
+    by_folder = data["by_folder"]
+
+    lines.append("# All Tests by Folder\n")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{folder}/ ({len(folder_tests)} tests)</h2></summary>\n"
+        )
+
+        # Group by backend within folder
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+            backend_tests = folder_by_backend.get(backend, [])
+            if not backend_tests:
+                continue
+
+            lines.append(f"### {backend} ({len(backend_tests)} tests)\n")
+            lines.append("| Test File | Suite | Est. Time | Status |")
+            lines.append("|-----------|-------|-----------|--------|")
+
+            for t in sorted(backend_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                status = (
+                    "Disabled"
+                    if t.disabled
+                    else ("Nightly" if t.nightly else "Per-Commit")
+                )
+                lines.append(
+                    f"| `{test_name}` | {t.suite} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_suite_section(data: dict) -> str:
+    """Generate the 'All Tests by Test Suite' section."""
+    lines = []
+    by_backend = data["by_backend"]
+
+    lines.append("# All Tests by Test Suite\n")
+
+    for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{backend} Backend ({b_enabled} enabled, {b_disabled} disabled)</h2></summary>\n"
+        )
+
+        # Group by suite within backend
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.suite].append(t)
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            s_enabled = sum(1 for t in suite_tests if not t.disabled)
+            s_disabled = sum(1 for t in suite_tests if t.disabled)
+            s_est_time = sum(t.est_time for t in suite_tests if not t.disabled)
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            suite_type = "Nightly" if is_nightly else "Per-Commit"
+            lines.append("<details>")
+            lines.append(
+                f"<summary><h3>{suite} ({s_enabled} enabled, {s_disabled} disabled) - {suite_type}</h3></summary>\n"
+            )
+            lines.append(f"*Estimated total time: {s_est_time:.0f}s*\n")
+
+            lines.append("| Test File | Folder | Est. Time | Status |")
+            lines.append("|-----------|--------|-----------|--------|")
+
+            for t in sorted(suite_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                folder = get_folder_name(t.filename)
+                if t.disabled:
+                    status = (
+                        f"Disabled: {t.disabled[:30]}..."
+                        if len(t.disabled) > 30
+                        else f"Disabled: {t.disabled}"
+                    )
+                else:
+                    status = "Nightly" if t.nightly else "Per-Commit"
+                lines.append(
+                    f"| `{test_name}` | {folder} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("\n</details>\n")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_markdown_report(tests: list[CIRegistry], section: str = "all") -> str:
+    """Generate markdown report for GitHub step summary."""
+    data = organize_test_data(tests)
+
+    if section == "summary":
+        return generate_summary_section(data)
+    elif section == "by-folder":
+        return generate_by_folder_section(data)
+    elif section == "by-suite":
+        return generate_by_suite_section(data)
+    else:  # "all"
+        parts = [
+            generate_summary_section(data),
+            "---",
+            generate_by_folder_section(data),
+            "---",
+            generate_by_suite_section(data),
+        ]
+        return "\n".join(parts)
+
+
+def generate_json_report(tests: list[CIRegistry]) -> str:
+    """Generate JSON report with detailed test listings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+
+    disabled_tests = [t for t in tests if t.disabled]
+
+    # Build structured data
+    data = {
+        "summary": {
+            "total": len(tests),
+            "enabled": len(tests) - len(disabled_tests),
+            "disabled": len(disabled_tests),
+        },
+        "tests_by_folder": {},
+        "tests_by_suite": {},
+        "backend_summary": {},
+        "folder_summary": {},
+        "disabled_tests": [],
+    }
+
+    # Section 1: Tests by Folder
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        data["tests_by_folder"][folder] = {
+            "total": len(folder_tests),
+            "backends": {},
+        }
+
+        for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+            backend_tests = folder_by_backend.get(backend, [])
+            if backend_tests:
+                data["tests_by_folder"][folder]["backends"][backend] = [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "suite": t.suite,
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                    }
+                    for t in sorted(backend_tests, key=lambda x: x.filename)
+                ]
+
+    # Section 2: Tests by Suite (Backend -> Suite)
+    for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.suite].append(t)
+
+        data["tests_by_suite"][backend] = {
+            "total": len(backend_tests),
+            "enabled": sum(1 for t in backend_tests if not t.disabled),
+            "disabled": sum(1 for t in backend_tests if t.disabled),
+            "suites": {},
+        }
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            data["tests_by_suite"][backend]["suites"][suite] = {
+                "total": len(suite_tests),
+                "enabled": sum(1 for t in suite_tests if not t.disabled),
+                "disabled": sum(1 for t in suite_tests if t.disabled),
+                "est_time": sum(t.est_time for t in suite_tests if not t.disabled),
+                "type": "nightly" if is_nightly else "per-commit",
+                "tests": [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "folder": get_folder_name(t.filename),
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                        "disabled_reason": t.disabled if t.disabled else None,
+                    }
+                    for t in sorted(suite_tests, key=lambda x: x.filename)
+                ],
+            }
+
+    # Backend summary
+    for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+        backend_tests = by_backend.get(backend, [])
+        if backend_tests:
+            data["backend_summary"][backend] = {
+                "total": len(backend_tests),
+                "enabled": sum(1 for t in backend_tests if not t.disabled),
+                "disabled": sum(1 for t in backend_tests if t.disabled),
+                "per_commit": sum(
+                    1 for t in backend_tests if not t.nightly and not t.disabled
+                ),
+                "nightly": sum(
+                    1 for t in backend_tests if t.nightly and not t.disabled
+                ),
+            }
+
+    # Folder summary
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        data["folder_summary"][folder] = {
+            "CUDA": sum(1 for t in folder_tests if t.backend == HWBackend.CUDA),
+            "AMD": sum(1 for t in folder_tests if t.backend == HWBackend.AMD),
+            "NPU": sum(1 for t in folder_tests if t.backend == HWBackend.NPU),
+            "CPU": sum(1 for t in folder_tests if t.backend == HWBackend.CPU),
+            "total": len(folder_tests),
+        }
+
+    # Disabled tests
+    for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+        data["disabled_tests"].append(
+            {
+                "filename": get_test_basename(t.filename),
+                "backend": t.backend.name,
+                "suite": t.suite,
+                "reason": t.disabled,
+            }
+        )
+
+    return json.dumps(data, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate CI coverage report")
+    parser.add_argument(
+        "--output-format",
+        choices=["markdown", "json"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    parser.add_argument(
+        "--section",
+        choices=["all", "summary", "by-folder", "by-suite"],
+        default="all",
+        help="Which section to output (default: all). Only applies to markdown format.",
+    )
+    parser.add_argument(
+        "--registered-dir",
+        default="test/registered",
+        help="Path to registered test directory",
+    )
+    args = parser.parse_args()
+
+    # Change to repo root if needed
+    script_dir = Path(__file__).parent.parent
+    repo_root = script_dir.parent.parent
+    os.chdir(repo_root)
+
+    tests = collect_all_tests(args.registered_dir)
+
+    if args.output_format == "markdown":
+        report = generate_markdown_report(tests, section=args.section)
+    else:
+        report = generate_json_report(tests)
+
+    print(report)
+
+    # Write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file and args.output_format == "markdown":
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/cleanup_hf_cache.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Clean up stale HuggingFace cache artifacts from previous failed downloads.
+
+This script removes incomplete marker files, temporary files, and lock files
+from the HuggingFace cache directory. These artifacts can accumulate from
+interrupted or failed downloads and may interfere with future downloads.
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import List
+
+try:
+    from huggingface_hub import constants
+
+    HF_HUB_AVAILABLE = True
+except ImportError:
+    print("Warning: huggingface_hub not available")
+    HF_HUB_AVAILABLE = False
+
+
+def get_hf_cache_dir() -> str:
+    """Get the HuggingFace cache directory."""
+    if HF_HUB_AVAILABLE:
+        return constants.HF_HUB_CACHE
+
+    # Fallback to environment variable or default
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    return os.path.join(hf_home, "hub")
+
+
+def find_stale_artifacts(cache_dir: str) -> List[Path]:
+    """
+    Find stale artifact files in the HuggingFace cache.
+
+    Args:
+        cache_dir: HuggingFace cache directory
+
+    Returns:
+        List of paths to stale artifact files
+    """
+    cache_path = Path(cache_dir)
+
+    if not cache_path.exists():
+        return []
+
+    # Patterns for stale files to clean up
+    patterns = [
+        "**/*.incomplete",  # Incomplete download markers
+        "**/*.tmp",  # Temporary files
+        "**/*.lock",  # Lock files from interrupted downloads
+    ]
+
+    stale_files = []
+    for pattern in patterns:
+        stale_files.extend(cache_path.glob(pattern))
+
+    return stale_files
+
+
+def cleanup_artifacts(artifacts: List[Path]) -> tuple[int, int]:
+    """
+    Remove stale artifact files.
+
+    Args:
+        artifacts: List of file paths to remove
+
+    Returns:
+        Tuple of (successful_removals, failed_removals)
+    """
+    successful = 0
+    failed = 0
+
+    for file_path in artifacts:
+        try:
+            file_path.unlink()
+            print(f"  Removed: {file_path}")
+            successful += 1
+        except Exception as e:
+            print(f"  Warning: Could not remove {file_path}: {e}")
+            failed += 1
+
+    return successful, failed
+
+
+def main() -> int:
+    """
+    Main cleanup logic.
+
+    Returns:
+        Always returns 0 (cleanup is best-effort and should not fail CI)
+    """
+    print("=" * 70)
+    print("HuggingFace Cache Cleanup")
+    print("=" * 70)
+
+    # Get cache directory
+    cache_dir = get_hf_cache_dir()
+    print(f"Cache directory: {cache_dir}")
+
+    if not os.path.exists(cache_dir):
+        print("Cache directory does not exist - nothing to clean")
+        return 0
+
+    print("-" * 70)
+
+    # Find stale artifacts
+    print("Scanning for stale artifacts...")
+    stale_artifacts = find_stale_artifacts(cache_dir)
+
+    if not stale_artifacts:
+        print("✓ No stale cache artifacts found")
+        return 0
+
+    # Clean up artifacts
+    print(f"Found {len(stale_artifacts)} stale artifact(s) to remove:")
+    successful, failed = cleanup_artifacts(stale_artifacts)
+
+    print("-" * 70)
+
+    # Summary
+    if failed > 0:
+        print(f"⚠ Cleaned up {successful} file(s), {failed} removal(s) failed")
+    else:
+        print(f"✓ Successfully cleaned up {successful} stale file(s)")
+
+    # Always return 0 - cleanup failures should not fail CI
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = main()
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        print(f"ERROR: Unexpected error during cleanup: {e}")
+        import traceback
+
+        traceback.print_exc()
+        # Still return 0 - cleanup failures should not fail CI
+        sys.exit(0)

third_party/sglang/scripts/ci/utils/diffusion/comparison_configs.json

@@ -0,0 +1,157 @@
+{
+    "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
+    "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
+    "cases": [
+        {
+            "id": "flux1_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.1-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --dit-layerwise-offload false",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "flux2_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.2-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --dit-layerwise-offload false",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_2512_t2i_1024",
+            "model": "Qwen/Qwen-Image-2512",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_edit_2511",
+            "model": "Qwen/Qwen-Image-Edit-2511",
+            "task": "image-edit",
+            "prompt": "Make the cat wear a red hat",
+            "reference_image": true,
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "zimage_turbo_t2i_1024",
+            "model": "Tongyi-MAI/Z-Image-Turbo",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_t2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_ti2v_5b_720p",
+            "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+            "task": "text-image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "ltx2_twostage_t2v",
+            "model": "Lightricks/LTX-2",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 768,
+            "height": 512,
+            "num_frames": 121,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_i2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+            "task": "image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        }
+    ]
+}

third_party/sglang/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -0,0 +1,836 @@
+"""Generate a Markdown dashboard for diffusion cross-framework comparisons.
+
+Reads current comparison results + historical data from sglang-ci-data repo
+and produces a Markdown report with tables and trend charts saved as PNG files.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
+        --results comparison-results.json \
+        --output dashboard.md \
+        --charts-dir comparison-charts/ \
+        --history-dir history/           # optional, local history JSONs
+        --fetch-history                  # fetch from GitHub API instead
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+# ---------------------------------------------------------------------------
+# History fetching (from sglang-ci-data repo via GitHub API)
+# ---------------------------------------------------------------------------
+
+CI_DATA_REPO_OWNER = "sglang-bot"
+CI_DATA_REPO_NAME = "sglang-ci-data"
+CI_DATA_BRANCH = "main"
+HISTORY_PREFIX = "diffusion-comparisons"
+MAX_HISTORY_RUNS = 14
+
+# Base URL for chart images pushed to sglang-ci-data
+CHARTS_RAW_BASE_URL = (
+    f"https://raw.githubusercontent.com/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+    f"/{CI_DATA_BRANCH}/{HISTORY_PREFIX}/charts"
+)
+
+
+def _github_get(url: str, token: str) -> dict | list | None:
+    """Simple GET to GitHub API."""
+    from urllib.error import HTTPError
+    from urllib.request import Request, urlopen
+
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    req = Request(url, headers=headers)
+    try:
+        with urlopen(req) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except HTTPError as e:
+        print(f"  Warning: GitHub API request failed ({e.code}): {url}")
+        return None
+    except Exception as e:
+        print(f"  Warning: GitHub API request error: {e}")
+        return None
+
+
+def fetch_history_from_github(token: str) -> list[dict]:
+    """Fetch recent comparison result JSONs from sglang-ci-data repo."""
+    print("Fetching historical comparison data from GitHub...")
+    url = (
+        f"https://api.github.com/repos/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+        f"/contents/{HISTORY_PREFIX}?ref={CI_DATA_BRANCH}"
+    )
+    listing = _github_get(url, token)
+    if not listing or not isinstance(listing, list):
+        print("  No historical data found.")
+        return []
+
+    # Filter JSON files and sort by name (date prefix) descending
+    json_files = sorted(
+        [f for f in listing if f["name"].endswith(".json")],
+        key=lambda f: f["name"],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+
+    history = []
+    for entry in json_files:
+        raw_url = entry.get("download_url")
+        if not raw_url:
+            continue
+        data = _github_get(raw_url, token)
+        if data and isinstance(data, dict):
+            history.append(data)
+    print(f"  Loaded {len(history)} historical run(s).")
+    return history
+
+
+def load_history_from_dir(history_dir: str) -> list[dict]:
+    """Load historical JSONs from a local directory."""
+    if not os.path.isdir(history_dir):
+        return []
+    files = sorted(
+        [f for f in os.listdir(history_dir) if f.endswith(".json")],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+    history = []
+    for fname in files:
+        try:
+            with open(os.path.join(history_dir, fname)) as f:
+                history.append(json.load(f))
+        except Exception:
+            pass
+    return history
+
+
+# ---------------------------------------------------------------------------
+# Dashboard generation
+# ---------------------------------------------------------------------------
+
+
+def _fmt_latency(val: float | None) -> str:
+    if val is None:
+        return "N/A"
+    return f"{val:.2f}"
+
+
+def _fmt_speedup(sglang_lat: float | None, other_lat: float | None) -> str:
+    if sglang_lat is None or other_lat is None or sglang_lat <= 0:
+        return "N/A"
+    ratio = other_lat / sglang_lat
+    return f"{ratio:.2f}x"
+
+
+def _short_date(ts: str) -> str:
+    """Extract short date from ISO timestamp."""
+    try:
+        dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+        return dt.strftime("%b %d")
+    except Exception:
+        return ts[:10]
+
+
+def _short_sha(sha: str) -> str:
+    return sha[:7] if sha and sha != "unknown" else "?"
+
+
+def _assess_risk(
+    cid: str,
+    current_cases: dict[str, dict[str, float | None]],
+    history: list[dict],
+    other_frameworks: list[str],
+) -> tuple[str, str]:
+    """Assess risk for a given case, returning (emoji, reason).
+
+    Rules (checked in order):
+    - N/A latency → ❌ broken
+    - History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression
+    - Competitor exists & SGLang slower → 🔴 competitive risk
+    - SGLang faster than all competitors by >20% → 🟢 strong advantage
+    - SGLang faster than all competitors by ≤20% → 🟡 moderate advantage
+    - Default → ✅ stable
+    """
+    sg_lat = current_cases.get(cid, {}).get("sglang")
+
+    # Broken: sglang latency is N/A
+    if sg_lat is None:
+        return "❌", f"{cid}: SGLang latency is N/A (broken)"
+
+    # Check regression against 3-run historical average
+    if history:
+        hist_lats: list[float] = []
+        for run in history[:3]:
+            run_cases = _extract_case_results(run)
+            h_lat = run_cases.get(cid, {}).get("sglang")
+            if h_lat is not None:
+                hist_lats.append(h_lat)
+        if hist_lats:
+            avg_3 = sum(hist_lats) / len(hist_lats)
+            if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05:
+                pct = (sg_lat - avg_3) / avg_3 * 100
+                return (
+                    "⚠️",
+                    f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg "
+                    f"({sg_lat:.2f}s vs {avg_3:.2f}s)",
+                )
+
+    # Check competitive risk
+    if other_frameworks:
+        competitor_lats: dict[str, float] = {}
+        for ofw in other_frameworks:
+            olat = current_cases.get(cid, {}).get(ofw)
+            if olat is not None:
+                competitor_lats[ofw] = olat
+
+        if competitor_lats:
+            # SGLang slower than any competitor?
+            for ofw, olat in competitor_lats.items():
+                if sg_lat > olat:
+                    return (
+                        "🔴",
+                        f"{cid}: SGLang slower than {ofw} "
+                        f"({sg_lat:.2f}s vs {olat:.2f}s)",
+                    )
+
+            # SGLang faster — check margin
+            min_competitor = min(competitor_lats.values())
+            advantage = (min_competitor - sg_lat) / min_competitor
+            if advantage > 0.20:
+                return "🟢", ""
+            else:
+                return "🟡", ""
+
+    # Default: stable
+    return "✅", ""
+
+
+def _trend_emoji(current: float | None, previous: float | None) -> str:
+    if current is None or previous is None:
+        return ""
+    diff_pct = (current - previous) / previous * 100
+    if diff_pct < -2:
+        return " :arrow_down:"  # faster (good)
+    elif diff_pct > 2:
+        return " :arrow_up:"  # slower (bad)
+    return " :left_right_arrow:"
+
+
+def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]:
+    """Extract {case_id: {framework: latency}} from a run."""
+    mapping: dict[str, dict[str, float | None]] = {}
+    for r in run_data.get("results", []):
+        cid = r["case_id"]
+        fw = r["framework"]
+        if cid not in mapping:
+            mapping[cid] = {}
+        mapping[cid][fw] = r.get("latency_s")
+    return mapping
+
+
+def _sanitize_filename(name: str) -> str:
+    """Sanitize a case ID to be a safe filename."""
+    return name.replace("/", "_").replace(" ", "_").replace(":", "_")
+
+
+def generate_dashboard(
+    current: dict,
+    history: list[dict],
+    charts_dir: str | None = None,
+) -> tuple[str, list[str]]:
+    """Generate full markdown dashboard.
+
+    Returns (markdown_string, alert_reasons) where alert_reasons is a list of
+    human-readable strings for cases that need attention (empty if all is well).
+
+    If charts_dir is provided, saves chart PNGs as files to that directory
+    and references them via raw.githubusercontent URLs. Otherwise, charts
+    are omitted.
+
+    Returns the markdown string.
+    """
+    lines: list[str] = []
+    lines.append("# Diffusion Cross-Framework Performance Dashboard\n")
+    ts = current.get("timestamp", datetime.now(timezone.utc).isoformat())
+    sha = current.get("commit_sha", "unknown")
+    lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
+
+    current_cases = _extract_case_results(current)
+    case_ids = list(current_cases.keys())
+
+    # ---- Regression detection ----
+    REGRESSION_THRESHOLD = 0.05  # 5%
+    regressions: list[str] = []
+    if history:
+        prev_cases = _extract_case_results(history[0])
+        for cid in case_ids:
+            for fw in ("sglang", "vllm-omni"):
+                cur = current_cases.get(cid, {}).get(fw)
+                prev = prev_cases.get(cid, {}).get(fw)
+                if cur and prev and prev > 0:
+                    pct = (cur - prev) / prev
+                    if pct > REGRESSION_THRESHOLD:
+                        regressions.append(
+                            f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s "
+                            f"(+{pct*100:.1f}%)"
+                        )
+
+    if regressions:
+        lines.append("> [!WARNING]\n> **Performance Regression Detected**\n>")
+        for reg in regressions:
+            lines.append(f"> - {reg}")
+        lines.append("\n")
+
+    # Discover all frameworks present in results
+    all_frameworks = []
+    seen_fw = set()
+    for r in current.get("results", []):
+        fw = r["framework"]
+        if fw not in seen_fw:
+            all_frameworks.append(fw)
+            seen_fw.add(fw)
+    # Ensure sglang is first
+    if "sglang" in all_frameworks:
+        all_frameworks.remove("sglang")
+        all_frameworks.insert(0, "sglang")
+    other_frameworks = [fw for fw in all_frameworks if fw != "sglang"]
+
+    # ---- Section 1: Cross-Framework Comparison (current run) ----
+    lines.append("## Cross-Framework Performance Comparison\n")
+
+    # Compute risk assessments for all cases
+    risk_map: dict[str, tuple[str, str]] = {}
+    for cid in case_ids:
+        risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
+
+    # Dynamic header
+    header = "| Model | Risk |"
+    sep = "|-------|------|"
+    for fw in all_frameworks:
+        header += f" {fw} (s) |"
+        sep += "---------|"
+    for ofw in other_frameworks:
+        header += f" vs {ofw} |"
+        sep += "---------|"
+    lines.append(header)
+    lines.append(sep)
+
+    # One row per case (deduplicated by case_id)
+    seen_cases = set()
+    for r in current.get("results", []):
+        cid = r["case_id"]
+        if cid in seen_cases:
+            continue
+        seen_cases.add(cid)
+
+        case_fws = current_cases.get(cid, {})
+        sg_lat = case_fws.get("sglang")
+
+        risk_emoji, _ = risk_map.get(cid, ("✅", ""))
+        row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |"
+        # Latency columns -- bold the fastest
+        lats = {fw: case_fws.get(fw) for fw in all_frameworks}
+        valid_lats = [v for v in lats.values() if v is not None]
+        min_lat = min(valid_lats) if valid_lats else None
+        for fw in all_frameworks:
+            lat = lats[fw]
+            if lat is not None and min_lat is not None and lat == min_lat:
+                row += f" **{_fmt_latency(lat)}** |"
+            else:
+                row += f" {_fmt_latency(lat)} |"
+        # Speedup columns
+        for ofw in other_frameworks:
+            row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
+        lines.append(row)
+
+    # ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ----
+    if history and other_frameworks:
+        lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
+
+        header = "| Date |"
+        sep = "|------|"
+        for cid in case_ids:
+            header += f" {cid} |"
+            sep += "---------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for run in all_runs:
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            row = f"| {date} |"
+            for cid in case_ids:
+                sg = run_cases.get(cid, {}).get("sglang")
+                vl = run_cases.get(cid, {}).get("vllm-omni")
+                row += f" {_fmt_speedup(sg, vl)} |"
+            lines.append(row)
+
+    # ---- Section 4: Matplotlib Trend Charts (saved as PNG files) ----
+    if history and charts_dir:
+        all_runs = list(reversed([current] + history))  # chronological order
+
+        def _chart_label(run: dict) -> str:
+            d = _short_date(run.get("timestamp", ""))
+            s = _short_sha(run.get("commit_sha", ""))
+            return f"{d}\n({s})"
+
+        try:
+            import matplotlib
+
+            matplotlib.use("Agg")
+            import matplotlib.pyplot as plt
+
+            os.makedirs(charts_dir, exist_ok=True)
+
+            # Per-case latency trend charts
+            for cid in case_ids:
+                labels = []
+                sg_vals = []
+                vl_vals = []
+                for run in all_runs:
+                    run_cases = _extract_case_results(run)
+                    sg = run_cases.get(cid, {}).get("sglang")
+                    vl = run_cases.get(cid, {}).get("vllm-omni")
+                    if sg is None:
+                        continue
+                    labels.append(_chart_label(run))
+                    sg_vals.append(sg)
+                    vl_vals.append(vl)
+
+                if not sg_vals:
+                    continue
+
+                has_vl = any(v is not None for v in vl_vals)
+                fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.2), 4))
+
+                # SGLang line
+                ax.plot(
+                    range(len(sg_vals)),
+                    sg_vals,
+                    "o-",
+                    color="#2563eb",
+                    linewidth=2,
+                    markersize=6,
+                    label="SGLang",
+                )
+                for i, v in enumerate(sg_vals):
+                    ax.annotate(
+                        f"{v:.2f}s",
+                        (i, v),
+                        textcoords="offset points",
+                        xytext=(0, 10),
+                        ha="center",
+                        fontsize=8,
+                        fontweight="bold",
+                        color="#2563eb",
+                    )
+
+                # vLLM-Omni line (if data exists)
+                if has_vl:
+                    vl_clean = [v if v is not None else float("nan") for v in vl_vals]
+                    ax.plot(
+                        range(len(vl_clean)),
+                        vl_clean,
+                        "s--",
+                        color="#dc2626",
+                        linewidth=2,
+                        markersize=5,
+                        label="vLLM-Omni",
+                    )
+                    for i, v in enumerate(vl_vals):
+                        if v is not None:
+                            ax.annotate(
+                                f"{v:.2f}s",
+                                (i, v),
+                                textcoords="offset points",
+                                xytext=(0, -14),
+                                ha="center",
+                                fontsize=8,
+                                color="#dc2626",
+                            )
+
+                ax.set_xticks(range(len(labels)))
+                ax.set_xticklabels(labels, fontsize=7)
+                ax.set_ylabel("Latency (s)")
+                ax.set_title(f"Latency Trend -- {cid}", fontsize=11, fontweight="bold")
+                ax.legend(loc="lower right", fontsize=8, framealpha=0.8)
+                ax.grid(True, alpha=0.3)
+                all_vals = sg_vals + [v for v in vl_vals if v is not None]
+                y_min = min(all_vals)
+                y_max = max(all_vals)
+                y_range = y_max - y_min if y_max > y_min else max(y_max * 0.1, 0.1)
+                ax.set_ylim(
+                    bottom=max(0, y_min - y_range * 0.3),
+                    top=y_max + y_range * 0.3,
+                )
+
+                filename = f"latency_{_sanitize_filename(cid)}.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append(f"\n### Latency Trend: {cid}\n")
+                lines.append(f"![Latency Trend {cid}]({chart_url})\n")
+
+            # Speedup trend chart (only if multiple frameworks)
+            if other_frameworks:
+                fig, ax = plt.subplots(figsize=(max(6, len(all_runs) * 1.2), 4))
+                colors = ["#2563eb", "#dc2626", "#16a34a", "#ea580c"]
+                for ci_idx, cid in enumerate(case_ids):
+                    speedups = []
+                    run_labels = []
+                    for run in all_runs:
+                        run_cases = _extract_case_results(run)
+                        sg = run_cases.get(cid, {}).get("sglang")
+                        vl = run_cases.get(cid, {}).get("vllm-omni")
+                        if sg and vl and sg > 0:
+                            speedups.append(vl / sg)
+                        else:
+                            speedups.append(None)
+                        run_labels.append(_chart_label(run))
+                    clean = [v if v is not None else float("nan") for v in speedups]
+                    ax.plot(
+                        range(len(clean)),
+                        clean,
+                        "o-",
+                        color=colors[ci_idx % len(colors)],
+                        linewidth=2,
+                        markersize=5,
+                        label=cid,
+                    )
+
+                ax.set_xticks(range(len(run_labels)))
+                ax.set_xticklabels(run_labels, fontsize=7)
+                ax.set_ylabel("Speedup (x)")
+                ax.set_title(
+                    "SGLang Speedup Over vLLM-Omni", fontsize=11, fontweight="bold"
+                )
+                ax.axhline(y=1.0, color="gray", linestyle=":", alpha=0.5)
+                ax.legend(loc="upper left", fontsize=7)
+                ax.grid(True, alpha=0.3)
+
+                filename = "speedup_trend.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append("\n### Speedup Trend (SGLang vs vLLM-Omni)\n")
+                lines.append(f"![Speedup Trend]({chart_url})\n")
+
+        except ImportError:
+            lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
+
+    # ---- SGLang Performance Trend (raw data table, at the end) ----
+    if history:
+        lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
+
+        header = "| Date | Commit |"
+        sep = "|------|--------|"
+        for cid in case_ids:
+            header += f" {cid} (s) |"
+            sep += "---------|"
+        header += " Trend |"
+        sep += "-------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for i, run in enumerate(all_runs):
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            sha_s = _short_sha(run.get("commit_sha", ""))
+            row = f"| {date} | `{sha_s}` |"
+            for cid in case_ids:
+                lat = run_cases.get(cid, {}).get("sglang")
+                row += f" {_fmt_latency(lat)} |"
+            if i + 1 < len(all_runs):
+                prev_cases = _extract_case_results(all_runs[i + 1])
+                emojis = []
+                for cid in case_ids:
+                    cur = run_cases.get(cid, {}).get("sglang")
+                    prev = prev_cases.get(cid, {}).get("sglang")
+                    emojis.append(_trend_emoji(cur, prev))
+                row += " ".join(emojis) + " |"
+            else:
+                row += " -- |"
+            lines.append(row)
+
+    # ---- Risk Notification ----
+    alert_cases = [
+        (cid, emoji, reason)
+        for cid, (emoji, reason) in risk_map.items()
+        if emoji in ("⚠️", "🔴", "❌")
+    ]
+    if alert_cases:
+        lines.append("\n> [!CAUTION]")
+        lines.append("> **Action Required — Performance Alert**")
+        lines.append(">")
+        lines.append("> The following cases need attention:")
+        for _cid, _emoji, reason in alert_cases:
+            lines.append(f"> - {reason}")
+        lines.append("")
+
+    # Footer
+    lines.append("\n---")
+    lines.append(
+        "*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
+    )
+
+    alert_reasons = [reason for _, _, reason in alert_cases]
+    return "\n".join(lines) + "\n", alert_reasons
+
+
+ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
+ALERT_LABEL = "perf-regression"
+
+
+ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
+
+
+def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
+    """Find the perf-regression tracker issue (open OR closed).
+
+    Returns (issue_number, is_open).  Prefers an open issue; if none,
+    returns the most recent closed one so it can be reopened.
+    """
+    import subprocess
+
+    for state in ("open", "closed"):
+        result = subprocess.run(
+            [
+                "gh",
+                "issue",
+                "list",
+                "--repo",
+                repo,
+                "--label",
+                ALERT_LABEL,
+                "--state",
+                state,
+                "--json",
+                "number",
+                "--limit",
+                "1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode != 0 or not result.stdout.strip():
+            continue
+        issues = json.loads(result.stdout)
+        if issues:
+            return str(issues[0]["number"]), state == "open"
+    return None, False
+
+
+def _create_alert_issue(alert_reasons: list[str]) -> None:
+    """Create or update the single perf-regression tracker issue.
+
+    Logic:
+    - If an open issue exists  → add a comment with the new alert.
+    - If a closed issue exists → reopen it, then add a comment.
+    - If no issue exists       → create one.
+
+    This guarantees at most one tracker issue ever exists.
+
+    Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
+    Falls back silently outside CI.
+    """
+    import subprocess
+
+    run_url = ""
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+    server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    if run_id:
+        run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
+
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    body_lines = [
+        f"## Performance Alert — {date}",
+        "",
+        "The nightly diffusion benchmark detected the following issue(s):",
+        "",
+    ]
+    for reason in alert_reasons:
+        body_lines.append(f"- {reason}")
+    if run_url:
+        body_lines += ["", f"**CI Run:** {run_url}"]
+    body = "\n".join(body_lines)
+
+    try:
+        existing, is_open = _find_alert_issue(repo)
+
+        if existing:
+            # Reopen if closed
+            if not is_open:
+                subprocess.run(
+                    [
+                        "gh",
+                        "issue",
+                        "reopen",
+                        existing,
+                        "--repo",
+                        repo,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                )
+                print(f"Reopened alert issue #{existing}")
+
+            # Add comment
+            result = subprocess.run(
+                [
+                    "gh",
+                    "issue",
+                    "comment",
+                    existing,
+                    "--repo",
+                    repo,
+                    "--body",
+                    body,
+                ],
+                capture_output=True,
+                text=True,
+                timeout=30,
+            )
+            if result.returncode == 0:
+                print(f"Commented on alert issue #{existing}")
+            else:
+                print(
+                    f"Warning: failed to comment on issue #{existing} "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+        else:
+            # Create a new issue
+            cmd = [
+                "gh",
+                "issue",
+                "create",
+                "--repo",
+                repo,
+                "--title",
+                ALERT_ISSUE_TITLE,
+                "--body",
+                body,
+                "--label",
+                ALERT_LABEL,
+            ]
+            for user in ALERT_ASSIGNEES:
+                cmd += ["--assignee", user]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            if result.returncode == 0:
+                print(f"Created alert issue: {result.stdout.strip()}")
+            else:
+                print(
+                    f"Warning: failed to create alert issue "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+    except FileNotFoundError:
+        print("Warning: `gh` CLI not found — skipping alert issue creation")
+    except Exception as e:
+        print(f"Warning: failed to create/update alert issue: {e}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate diffusion cross-framework comparison dashboard"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json from current run",
+    )
+    parser.add_argument(
+        "--output",
+        default="dashboard.md",
+        help="Output markdown file path",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default="comparison-charts",
+        help="Directory to save chart PNG files (default: comparison-charts/)",
+    )
+    parser.add_argument(
+        "--history-dir",
+        default=None,
+        help="Local directory containing historical comparison JSONs",
+    )
+    parser.add_argument(
+        "--fetch-history",
+        action="store_true",
+        help="Fetch history from sglang-ci-data GitHub repo",
+    )
+    parser.add_argument(
+        "--step-summary",
+        action="store_true",
+        help="Also write to $GITHUB_STEP_SUMMARY",
+    )
+
+    args = parser.parse_args()
+
+    # Load current results
+    with open(args.results) as f:
+        current = json.load(f)
+    print(f"Loaded current results: {len(current.get('results', []))} entries")
+
+    # Load history
+    history: list[dict] = []
+    if args.fetch_history:
+        token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+            "GITHUB_TOKEN"
+        )
+        if token:
+            history = fetch_history_from_github(token)
+        else:
+            print("Warning: No GitHub token available, skipping history fetch")
+    elif args.history_dir:
+        history = load_history_from_dir(args.history_dir)
+        print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
+
+    # Generate dashboard
+    markdown, alert_reasons = generate_dashboard(
+        current, history, charts_dir=args.charts_dir
+    )
+
+    # Write output
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.output, "w") as f:
+        f.write(markdown)
+    print(f"Dashboard written to {args.output}")
+
+    # Write to GitHub Step Summary
+    if args.step_summary:
+        summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary_file:
+            with open(summary_file, "a") as f:
+                f.write(markdown)
+            print("Dashboard appended to $GITHUB_STEP_SUMMARY")
+        else:
+            print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
+
+    # Create GitHub Issue for performance alerts (so assignees get notified)
+    if alert_reasons:
+        _create_alert_issue(alert_reasons)
+    else:
+        print("No performance alerts — skipping issue creation.")
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/diffusion/publish_comparison_results.py

@@ -0,0 +1,231 @@
+"""Publish diffusion comparison results to sglang-bot/sglang-ci-data repo.
+
+Pushes comparison-results.json, dashboard.md, and chart PNG files to the
+ci-data repository for historical tracking. Chart PNGs are stored under
+diffusion-comparisons/charts/ so they can be referenced via
+raw.githubusercontent URLs in the dashboard markdown (GitHub Step Summary
+blocks data: URIs).
+
+Usage:
+    python3 scripts/ci/utils/diffusion/publish_comparison_results.py \
+        --results comparison-results.json \
+        --dashboard dashboard.md \
+        --charts-dir comparison-charts/
+"""
+
+import argparse
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+# Repository configuration
+REPO_OWNER = "sglang-bot"
+REPO_NAME = "sglang-ci-data"
+BRANCH = "main"
+STORAGE_PREFIX = "diffusion-comparisons"
+
+
+def _collect_chart_files(charts_dir: str) -> list[tuple[str, bytes]]:
+    """Collect PNG chart files from directory for upload."""
+    files: list[tuple[str, bytes]] = []
+    if not charts_dir or not os.path.isdir(charts_dir):
+        return files
+
+    for entry in sorted(os.listdir(charts_dir)):
+        if not entry.lower().endswith(".png"):
+            continue
+        full_path = os.path.join(charts_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        # Store charts under diffusion-comparisons/charts/
+        repo_path = f"{STORAGE_PREFIX}/charts/{entry}"
+        files.append((repo_path, content))
+
+    return files
+
+
+def publish_comparison(
+    results_path: str,
+    dashboard_path: str | None = None,
+    charts_dir: str | None = None,
+) -> None:
+    """Publish comparison results, dashboard, and charts to ci-data repo."""
+    token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+        "GITHUB_TOKEN"
+    )
+    if not token:
+        print("Error: GH_PAT_FOR_NIGHTLY_CI_DATA or GITHUB_TOKEN not set")
+        sys.exit(1)
+
+    run_id = os.environ.get("GITHUB_RUN_ID", "local")
+    run_number = os.environ.get("GITHUB_RUN_NUMBER", "0")
+
+    # Verify permissions
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("Warning: Rate limited, skipping publish")
+        return
+    elif not perm:
+        print("Error: Token permission verification failed")
+        sys.exit(1)
+
+    # Prepare files to upload
+    files_to_upload: list[tuple[str, bytes]] = []
+
+    # Results JSON: stored with date prefix for chronological ordering
+    date_prefix = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    results_target = f"{STORAGE_PREFIX}/{date_prefix}_{run_id}.json"
+    with open(results_path, "rb") as f:
+        files_to_upload.append((results_target, f.read()))
+
+    # Dashboard markdown: always overwrite latest
+    if dashboard_path and os.path.exists(dashboard_path):
+        dashboard_target = f"{STORAGE_PREFIX}/dashboard.md"
+        with open(dashboard_path, "rb") as f:
+            files_to_upload.append((dashboard_target, f.read()))
+
+    # Chart PNG files
+    chart_files = _collect_chart_files(charts_dir)
+    if chart_files:
+        print(f"Found {len(chart_files)} chart PNG(s) to upload")
+        files_to_upload.extend(chart_files)
+
+    print(f"Publishing {len(files_to_upload)} file(s) to {REPO_OWNER}/{REPO_NAME}")
+
+    # Create blobs
+    try:
+        tree_items = create_blobs(REPO_OWNER, REPO_NAME, files_to_upload, token)
+    except Exception as e:
+        if is_rate_limit_error(e):
+            print("Warning: Rate limited during blob creation, skipping")
+            return
+        if is_permission_error(e):
+            print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+            sys.exit(1)
+        raise
+
+    # Commit with retry (handle concurrent writes)
+    max_retries = 5
+    retry_delay = 5
+
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+
+            commit_msg = (
+                f"Diffusion comparison results for run {run_id} (#{run_number})"
+            )
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully published comparison results (commit {commit_sha[:7]})"
+            )
+            return
+
+        except Exception as e:
+            is_retryable = False
+            if hasattr(e, "error_body"):
+                body = getattr(e, "error_body", "")
+                if "Update is not a fast forward" in body:
+                    is_retryable = True
+                elif "Object does not exist" in body:
+                    is_retryable = True
+
+            from urllib.error import HTTPError
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+
+            if is_rate_limit_error(e):
+                print("Warning: Rate limited, skipping publish")
+                return
+
+            if is_permission_error(e):
+                print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {retry_delay}s..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish diffusion comparison results to sglang-ci-data"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json",
+    )
+    parser.add_argument(
+        "--dashboard",
+        default=None,
+        help="Path to dashboard.md (optional)",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default=None,
+        help="Directory containing chart PNG files to upload (optional)",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.results):
+        print(f"Error: Results file not found: {args.results}")
+        sys.exit(1)
+
+    publish_comparison(
+        results_path=args.results,
+        dashboard_path=args.dashboard,
+        charts_dir=args.charts_dir,
+    )
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/diffusion/publish_diffusion_gt.py

@@ -0,0 +1,166 @@
+"""
+Publish diffusion CI ground-truth images to sglang-bot/sglang-ci-data
+via the GitHub API (same pattern as publish_traces.py).
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+REPO_OWNER = "sglang-bot"
+REPO_NAME = "sglang-ci-data"
+BRANCH = "main"
+TARGET_DIR = "diffusion-ci/consistency_gt"
+
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
+
+
+def collect_images(source_dir):
+    """Collect image files from source_dir and return list of (repo_path, content) tuples."""
+    files = []
+    for entry in sorted(os.listdir(source_dir)):
+        ext = os.path.splitext(entry)[1].lower()
+        if ext not in IMAGE_EXTENSIONS:
+            continue
+        full_path = os.path.join(source_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        repo_path = f"{TARGET_DIR}/{entry}"
+        files.append((repo_path, content))
+    return files
+
+
+def publish(source_dir):
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    files_to_upload = collect_images(source_dir)
+    if not files_to_upload:
+        print(f"No image files found in {source_dir}")
+        return
+
+    print(
+        f"Found {len(files_to_upload)} image(s) to upload to {REPO_OWNER}/{REPO_NAME}/{TARGET_DIR}"
+    )
+
+    # Verify token
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("GitHub API rate-limited, skipping upload.")
+        return
+    if not perm:
+        print("Token permission verification failed.")
+        sys.exit(1)
+
+    # Create blobs
+    try:
+        tree_items = create_blobs(REPO_OWNER, REPO_NAME, files_to_upload, token)
+    except Exception as e:
+        if is_rate_limit_error(e):
+            print("Rate-limited during blob creation, skipping.")
+            return
+        if is_permission_error(e):
+            print(
+                f"ERROR: Token lacks write permission to {REPO_OWNER}/{REPO_NAME}. "
+                "Update GH_PAT_FOR_NIGHTLY_CI_DATA with a token that has contents:write."
+            )
+            sys.exit(1)
+        raise
+
+    # Commit with retry (handle concurrent pushes)
+    max_retries = 5
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+            commit_msg = f"diffusion-ci: update consistency_gt images ({len(files_to_upload)} files) [automated]"
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully pushed {len(files_to_upload)} images (commit {commit_sha[:10]})"
+            )
+            return
+        except Exception as e:
+            if is_rate_limit_error(e):
+                print("Rate-limited, skipping.")
+                return
+            if is_permission_error(e):
+                print(f"ERROR: permission denied to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            retryable = False
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    retryable = True
+                elif "Object does not exist" in e.error_body:
+                    retryable = True
+
+            from urllib.error import HTTPError
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                retryable = True
+
+            if retryable and attempt < max_retries - 1:
+                import time
+
+                wait = 2**attempt
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {wait}s..."
+                )
+                time.sleep(wait)
+            else:
+                print(f"Failed after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish diffusion GT images to GitHub"
+    )
+    parser.add_argument(
+        "--source-dir", required=True, help="Directory containing GT images"
+    )
+    args = parser.parse_args()
+    publish(args.source_dir)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/diffusion/run_comparison.py

@@ -0,0 +1,951 @@
+"""Cross-framework comparison benchmark for diffusion serving.
+
+Launches servers (SGLang, vLLM-Omni, LightX2V) for each test case, sends a
+single request, measures end-to-end latency, and writes comparison-results.json.
+
+Usage:
+    # Full run (requires GPU)
+    python3 scripts/ci/utils/diffusion/run_comparison.py
+
+    # Dry-run (config parsing + command preview only)
+    python3 scripts/ci/utils/diffusion/run_comparison.py --dry-run
+
+    # Run only specific case(s)
+    python3 scripts/ci/utils/diffusion/run_comparison.py --case-ids flux1_dev_t2i_1024
+
+    # Run only specific framework(s)
+    python3 scripts/ci/utils/diffusion/run_comparison.py --frameworks sglang
+"""
+
+import argparse
+import base64
+import io
+import json
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import requests
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+CONFIGS_PATH = Path(__file__).parent / "comparison_configs.json"
+INSTALL_SCRIPT = Path(__file__).parents[1] / "install_comparison_frameworks.sh"
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 30000
+HEALTH_TIMEOUT = (
+    2400  # seconds (40 min — FLUX.2-dev needs ~10 min download + torch.compile)
+)
+REQUEST_TIMEOUT = 1200  # seconds
+GPU_CLEAR_WAIT = 15  # seconds between framework runs
+
+# Frameworks that need separate installation (conflict with sglang's deps)
+INSTALLABLE_FRAMEWORKS = {"vllm-omni", "lightx2v"}
+
+# Cached reference image (downloaded once)
+_cached_ref_image: bytes | None = None
+_cached_ref_image_path: str | None = None
+
+
+# ---------------------------------------------------------------------------
+# Server lifecycle — command builders
+# ---------------------------------------------------------------------------
+
+
+def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
+    cmd = [
+        "sglang",
+        "serve",
+        "--model-path",
+        case["model"],
+        "--port",
+        str(port),
+        "--host",
+        DEFAULT_HOST,
+    ]
+    if case["num_gpus"] > 1:
+        cmd += ["--num-gpus", str(case["num_gpus"])]
+    if fw_cfg.get("serve_args", "").strip():
+        cmd += fw_cfg["serve_args"].strip().split()
+    return cmd
+
+
+def _build_vllm_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
+    cmd = [
+        "vllm",
+        "serve",
+        case["model"],
+        "--omni",
+        "--port",
+        str(port),
+        "--host",
+        DEFAULT_HOST,
+    ]
+    if fw_cfg.get("serve_args", "").strip():
+        cmd += fw_cfg["serve_args"].strip().split()
+    return cmd
+
+
+def _resolve_hf_model_path(model_id: str) -> str:
+    """Resolve a HuggingFace model ID to a local cache path, or return as-is."""
+    if os.path.isdir(model_id):
+        return model_id
+    try:
+        from huggingface_hub import snapshot_download
+
+        path = snapshot_download(model_id)
+        print(f"  Resolved {model_id} -> {path}")
+        return path
+    except Exception:
+        return model_id
+
+
+def _write_lightx2v_config(case: dict) -> str:
+    """Write a minimal LightX2V config JSON and return its path."""
+    cfg = {
+        "infer_steps": case.get("num_inference_steps", 50),
+        "guidance_scale": case.get("guidance_scale", 4.0),
+        "seed": case.get("seed", 42),
+    }
+    if "num_frames" in case:
+        cfg["target_video_length"] = case["num_frames"]
+    if "height" in case:
+        cfg["height"] = case["height"]
+    if "width" in case:
+        cfg["width"] = case["width"]
+
+    config_path = os.path.join(
+        tempfile.gettempdir(), f"lightx2v_config_{case['id']}.json"
+    )
+    with open(config_path, "w") as f:
+        json.dump(cfg, f)
+    return config_path
+
+
+def _build_lightx2v_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
+    """Build LightX2V server launch command.
+
+    Single GPU:  python -m lightx2v.server --model_path ... --model_cls ... --task ... --port ...
+    Multi GPU:   torchrun --nproc_per_node=N -m lightx2v.server ...
+
+    LightX2V requires a local model path and a config JSON with infer params.
+    """
+    model_cls = fw_cfg["model_cls"]
+    task = fw_cfg["lightx2v_task"]
+    num_gpus = case["num_gpus"]
+    model_path = _resolve_hf_model_path(case["model"])
+    config_path = _write_lightx2v_config(case)
+
+    server_args = [
+        "--model_path",
+        model_path,
+        "--model_cls",
+        model_cls,
+        "--task",
+        task,
+        "--config_json",
+        config_path,
+        "--host",
+        DEFAULT_HOST,
+        "--port",
+        str(port),
+    ]
+    if fw_cfg.get("serve_args", "").strip():
+        server_args += fw_cfg["serve_args"].strip().split()
+
+    if num_gpus > 1:
+        cmd = [
+            "torchrun",
+            f"--nproc_per_node={num_gpus}",
+            "-m",
+            "lightx2v.server",
+        ] + server_args
+    else:
+        cmd = ["python3", "-m", "lightx2v.server"] + server_args
+
+    return cmd
+
+
+def build_server_cmd(framework: str, case: dict, fw_cfg: dict, port: int) -> list[str]:
+    builders = {
+        "sglang": _build_sglang_cmd,
+        "vllm-omni": _build_vllm_cmd,
+        "lightx2v": _build_lightx2v_cmd,
+    }
+    builder = builders.get(framework)
+    if builder is None:
+        raise ValueError(f"Unknown framework: {framework}")
+    return builder(case, fw_cfg, port)
+
+
+# ---------------------------------------------------------------------------
+# Server lifecycle — health check & cleanup
+# ---------------------------------------------------------------------------
+
+# Health check endpoints per framework
+HEALTH_ENDPOINTS = {
+    "sglang": "/health",
+    "vllm-omni": "/health",
+    "lightx2v": "/v1/service/status",
+}
+
+
+def wait_for_health(
+    base_url: str, framework: str = "sglang", timeout: int = HEALTH_TIMEOUT
+) -> None:
+    """Poll health endpoint until 200, then verify model is loaded."""
+    endpoint = HEALTH_ENDPOINTS.get(framework, "/health")
+    health_url = f"{base_url}{endpoint}"
+    print(f"  Waiting for server at {health_url} ...")
+    start = time.time()
+    while True:
+        try:
+            resp = requests.get(health_url, timeout=2)
+            if resp.status_code == 200:
+                break
+        except requests.exceptions.RequestException:
+            pass
+        if time.time() - start > timeout:
+            raise TimeoutError(
+                f"Server at {health_url} did not start within {timeout}s"
+            )
+        time.sleep(2)
+
+    # For SGLang, /health can return 200 before model routes are registered.
+    # Poll /v1/models to confirm the model is fully loaded.
+    if framework == "sglang":
+        models_url = f"{base_url}/v1/models"
+        while True:
+            try:
+                resp = requests.get(models_url, timeout=5)
+                if resp.status_code == 200:
+                    break
+            except requests.exceptions.RequestException:
+                pass
+            if time.time() - start > timeout:
+                raise TimeoutError(f"Model at {models_url} not ready within {timeout}s")
+            time.sleep(2)
+
+    elapsed = time.time() - start
+    print(f"  Server ready in {elapsed:.1f}s")
+
+
+KILLALL_SCRIPT = Path(__file__).parents[3] / "killall_sglang.sh"
+
+
+def kill_server(proc: subprocess.Popen) -> None:
+    """Kill server process tree and clean up GPU processes."""
+    if proc.poll() is not None:
+        return
+    try:
+        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+    except (ProcessLookupError, PermissionError):
+        pass
+    try:
+        proc.wait(timeout=30)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+        except (ProcessLookupError, PermissionError):
+            pass
+        proc.wait(timeout=10)
+    # Use killall_sglang.sh for thorough cleanup (esp. multi-GPU workers)
+    if KILLALL_SCRIPT.exists():
+        subprocess.run(
+            ["bash", str(KILLALL_SCRIPT)],
+            timeout=30,
+            capture_output=True,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Reference image helpers
+# ---------------------------------------------------------------------------
+
+
+def _get_ref_image_bytes(config: dict) -> bytes:
+    """Download and cache the shared test reference image."""
+    global _cached_ref_image
+    if _cached_ref_image is not None:
+        return _cached_ref_image
+    url = config.get("test_image_url", "")
+    if not url:
+        raise RuntimeError("No test_image_url in config for image-conditioned case")
+    print(f"  Downloading reference image from {url} ...")
+    resp = requests.get(url, timeout=60)
+    resp.raise_for_status()
+    _cached_ref_image = resp.content
+    return _cached_ref_image
+
+
+def _get_ref_image_b64(config: dict) -> str:
+    """Get reference image as base64 string."""
+    return base64.b64encode(_get_ref_image_bytes(config)).decode("utf-8")
+
+
+def _get_ref_image_path(config: dict) -> str:
+    """Save reference image to a temp file and return path."""
+    global _cached_ref_image_path
+    if _cached_ref_image_path and os.path.exists(_cached_ref_image_path):
+        return _cached_ref_image_path
+    data = _get_ref_image_bytes(config)
+    fd, path = tempfile.mkstemp(suffix=".png")
+    with os.fdopen(fd, "wb") as f:
+        f.write(data)
+    _cached_ref_image_path = path
+    return path
+
+
+# ---------------------------------------------------------------------------
+# Request helpers — SGLang (OpenAI-compatible)
+# ---------------------------------------------------------------------------
+
+
+def _build_sglang_payload(case: dict) -> dict:
+    """Build common SGLang request payload."""
+    payload = {
+        "model": case["model"],
+        "prompt": case["prompt"],
+        "size": f"{case['width']}x{case['height']}",
+        "n": 1,
+        "response_format": "b64_json",
+    }
+    for key in ("num_inference_steps", "guidance_scale", "seed", "num_frames"):
+        if key in case:
+            payload[key] = case[key]
+    return payload
+
+
+def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None:
+    """Read total_duration_ms from a perf dump JSON written by the server.
+
+    The server writes the file asynchronously after the HTTP response,
+    so we poll briefly.
+    """
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            with open(perf_dump_path) as f:
+                data = json.load(f)
+            total_ms = data.get("total_duration_ms")
+            if total_ms is not None:
+                return total_ms / 1000.0
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+        time.sleep(0.5)
+    return None
+
+
+def send_image_request_sglang(
+    base_url: str, case: dict, perf_dump_path: str | None = None
+) -> float:
+    """Send a single T2I request via SGLang's /v1/images/generations."""
+    payload = _build_sglang_payload(case)
+    if perf_dump_path:
+        payload["perf_dump_path"] = perf_dump_path
+
+    start = time.time()
+    resp = requests.post(
+        f"{base_url}/v1/images/generations",
+        json=payload,
+        timeout=REQUEST_TIMEOUT,
+    )
+    client_latency = time.time() - start
+    resp.raise_for_status()
+    data = resp.json()
+    if "data" not in data or len(data["data"]) == 0:
+        raise RuntimeError(f"Image request returned no data: {data}")
+
+    if perf_dump_path:
+        server_latency = _read_perf_dump(perf_dump_path)
+        if server_latency is not None:
+            print(
+                f"  Image generated in {server_latency:.2f}s (server-side), "
+                f"client={client_latency:.2f}s"
+            )
+            return server_latency
+    print(f"  Image generated in {client_latency:.2f}s")
+    return client_latency
+
+
+def send_video_request_sglang(
+    base_url: str, case: dict, perf_dump_path: str | None = None
+) -> float:
+    """Send a single T2V request via SGLang's /v1/videos (async)."""
+    payload = _build_sglang_payload(case)
+    if perf_dump_path:
+        payload["perf_dump_path"] = perf_dump_path
+
+    start = time.time()
+
+    # Submit job
+    resp = requests.post(
+        f"{base_url}/v1/videos",
+        json=payload,
+        timeout=REQUEST_TIMEOUT,
+    )
+    resp.raise_for_status()
+    job = resp.json()
+    job_id = job.get("id")
+    if not job_id:
+        raise RuntimeError(f"Video submit returned no job id: {job}")
+
+    # Poll for completion
+    poll_url = f"{base_url}/v1/videos/{job_id}"
+    while True:
+        time.sleep(1)
+        poll_resp = requests.get(poll_url, timeout=30)
+        poll_resp.raise_for_status()
+        poll_data = poll_resp.json()
+        status = poll_data.get("status")
+        if status == "completed":
+            break
+        elif status == "failed":
+            raise RuntimeError(f"Video generation failed: {poll_data}")
+        if time.time() - start > REQUEST_TIMEOUT:
+            raise TimeoutError(f"Video generation timed out after {REQUEST_TIMEOUT}s")
+
+    client_latency = time.time() - start
+
+    if perf_dump_path:
+        server_latency = _read_perf_dump(perf_dump_path)
+        if server_latency is not None:
+            print(
+                f"  Video generated in {server_latency:.2f}s (server-side), "
+                f"client={client_latency:.2f}s"
+            )
+            return server_latency
+    print(f"  Video generated in {client_latency:.2f}s")
+    return client_latency
+
+
+def send_image_conditioned_request_sglang(
+    base_url: str, case: dict, config: dict, perf_dump_path: str | None = None
+) -> float:
+    """Send an image-conditioned request (edit/I2V/TI2V) via SGLang multipart API."""
+    task = case["task"]
+    ref_bytes = _get_ref_image_bytes(config)
+
+    # Build multipart form — field name depends on endpoint:
+    # image edits use "image", video (I2V/TI2V) uses "input_reference"
+    if task in ("image-to-video", "text-image-to-video"):
+        file_field = "input_reference"
+    else:
+        file_field = "image"
+    files = {file_field: ("ref.png", io.BytesIO(ref_bytes), "image/png")}
+    data = {
+        "model": case["model"],
+        "prompt": case["prompt"],
+        "size": f"{case['width']}x{case['height']}",
+        "n": "1",
+        "response_format": "b64_json",
+    }
+    for key in ("num_inference_steps", "guidance_scale", "seed", "num_frames"):
+        if key in case:
+            data[key] = str(case[key])
+    if perf_dump_path:
+        data["perf_dump_path"] = perf_dump_path
+    # Choose endpoint based on task
+    if task in ("image-edit", "image-to-image"):
+        endpoint = "/v1/images/edits"
+    elif task in ("image-to-video", "text-image-to-video"):
+        endpoint = "/v1/videos"
+    else:
+        endpoint = "/v1/images/generations"
+
+    start = time.time()
+    resp = requests.post(
+        f"{base_url}{endpoint}",
+        files=files,
+        data=data,
+        timeout=REQUEST_TIMEOUT,
+    )
+
+    # For video endpoints, need to poll
+    if task in ("image-to-video", "text-image-to-video"):
+        resp.raise_for_status()
+        job = resp.json()
+        job_id = job.get("id")
+        if not job_id:
+            raise RuntimeError(f"Video submit returned no job id: {job}")
+        poll_url = f"{base_url}/v1/videos/{job_id}"
+        while True:
+            time.sleep(1)
+            poll_resp = requests.get(poll_url, timeout=30)
+            poll_resp.raise_for_status()
+            poll_data = poll_resp.json()
+            status = poll_data.get("status")
+            if status == "completed":
+                break
+            elif status == "failed":
+                raise RuntimeError(f"Video generation failed: {poll_data}")
+            if time.time() - start > REQUEST_TIMEOUT:
+                raise TimeoutError(f"Timed out after {REQUEST_TIMEOUT}s")
+    else:
+        resp.raise_for_status()
+
+    client_latency = time.time() - start
+
+    if perf_dump_path:
+        server_latency = _read_perf_dump(perf_dump_path)
+        if server_latency is not None:
+            print(
+                f"  Generated in {server_latency:.2f}s (server-side), "
+                f"client={client_latency:.2f}s"
+            )
+            return server_latency
+    print(f"  Generated in {client_latency:.2f}s (sglang, image-conditioned)")
+    return client_latency
+
+
+# ---------------------------------------------------------------------------
+# Request helpers — vLLM-Omni
+# ---------------------------------------------------------------------------
+
+
+def send_request_vllm_omni(base_url: str, case: dict, config: dict) -> float:
+    """Send request via vLLM-Omni's /v1/chat/completions endpoint."""
+    extra_body = {
+        "height": case["height"],
+        "width": case["width"],
+        "num_inference_steps": case.get("num_inference_steps", 50),
+        "guidance_scale": case.get("guidance_scale", 4.0),
+        "seed": case.get("seed", 42),
+    }
+    if "num_frames" in case:
+        extra_body["num_frames"] = case["num_frames"]
+
+    # Build message content (text or text+image)
+    content: list[dict] | str = case["prompt"]
+    if case.get("reference_image"):
+        ref_b64 = _get_ref_image_b64(config)
+        content = [
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/png;base64,{ref_b64}"},
+            },
+            {"type": "text", "text": case["prompt"]},
+        ]
+
+    payload = {
+        "model": case["model"],
+        "messages": [{"role": "user", "content": content}],
+        "extra_body": extra_body,
+    }
+
+    start = time.time()
+    resp = requests.post(
+        f"{base_url}/v1/chat/completions",
+        json=payload,
+        timeout=REQUEST_TIMEOUT,
+    )
+    latency = time.time() - start
+    resp.raise_for_status()
+    data = resp.json()
+    choices = data.get("choices", [])
+    if not choices:
+        raise RuntimeError(f"vLLM-Omni request returned no choices: {data}")
+    print(f"  Generated in {latency:.2f}s (vllm-omni)")
+    return latency
+
+
+# ---------------------------------------------------------------------------
+# Request helpers — LightX2V
+# ---------------------------------------------------------------------------
+
+
+def send_request_lightx2v(base_url: str, case: dict, config: dict) -> float:
+    """Send request via LightX2V's async task API."""
+    task = case["task"]
+    if task in ("text-to-image", "image-edit"):
+        endpoint = "/v1/tasks/image"
+    else:
+        endpoint = "/v1/tasks/video"
+
+    payload = {
+        "prompt": case["prompt"],
+        "seed": case.get("seed", 42),
+        "infer_steps": case.get("num_inference_steps", 50),
+    }
+    # LightX2V uses target_video_length for frames, height/width directly
+    if "num_frames" in case:
+        payload["target_video_length"] = case["num_frames"]
+    if "height" in case:
+        payload["height"] = case["height"]
+    if "width" in case:
+        payload["width"] = case["width"]
+    if "guidance_scale" in case:
+        payload["guidance_scale"] = case["guidance_scale"]
+    # Image-conditioned: LightX2V accepts image_path (URL or local path)
+    if case.get("reference_image"):
+        payload["image_path"] = config.get("test_image_url", "")
+
+    start = time.time()
+
+    # Submit task
+    resp = requests.post(
+        f"{base_url}{endpoint}",
+        json=payload,
+        timeout=REQUEST_TIMEOUT,
+    )
+    resp.raise_for_status()
+    task_data = resp.json()
+    task_id = task_data.get("task_id")
+    if not task_id:
+        raise RuntimeError(f"LightX2V submit returned no task_id: {task_data}")
+
+    # Poll for completion
+    poll_url = f"{base_url}/v1/tasks/{task_id}/status"
+    while True:
+        time.sleep(1)
+        poll_resp = requests.get(poll_url, timeout=30)
+        poll_resp.raise_for_status()
+        poll_data = poll_resp.json()
+        status = poll_data.get("task_status", "").upper()
+        if status == "COMPLETED":
+            break
+        elif status in ("FAILED", "CANCELLED"):
+            raise RuntimeError(f"LightX2V task {status}: {poll_data}")
+        if time.time() - start > REQUEST_TIMEOUT:
+            raise TimeoutError(f"LightX2V task timed out after {REQUEST_TIMEOUT}s")
+
+    latency = time.time() - start
+    print(f"  Generated in {latency:.2f}s (lightx2v)")
+    return latency
+
+
+# ---------------------------------------------------------------------------
+# Unified request dispatcher
+# ---------------------------------------------------------------------------
+
+
+def send_request(
+    base_url: str,
+    case: dict,
+    framework: str = "sglang",
+    config: dict | None = None,
+    perf_dump_path: str | None = None,
+) -> float:
+    config = config or {}
+    if framework == "vllm-omni":
+        return send_request_vllm_omni(base_url, case, config)
+    elif framework == "lightx2v":
+        return send_request_lightx2v(base_url, case, config)
+    # SGLang — use OpenAI-compatible endpoints with optional perf log
+    task = case["task"]
+    if case.get("reference_image"):
+        return send_image_conditioned_request_sglang(
+            base_url, case, config, perf_dump_path
+        )
+    elif task == "text-to-image":
+        return send_image_request_sglang(base_url, case, perf_dump_path)
+    elif task == "text-to-video":
+        return send_video_request_sglang(base_url, case, perf_dump_path)
+    else:
+        raise ValueError(f"Unknown task type: {task}")
+
+
+# ---------------------------------------------------------------------------
+# Main orchestrator
+# ---------------------------------------------------------------------------
+
+
+def run_single(
+    case: dict,
+    framework: str,
+    fw_cfg: dict,
+    port: int,
+    log_dir: Path,
+    config: dict | None = None,
+) -> dict:
+    """Run a single (case, framework) combination. Returns result dict."""
+    result = {
+        "case_id": case["id"],
+        "framework": framework,
+        "model": case["model"],
+        "task": case["task"],
+        "latency_s": None,
+        "error": None,
+    }
+
+    cmd = build_server_cmd(framework, case, fw_cfg, port)
+    print(f"\n  Command: {' '.join(cmd)}")
+
+    env = os.environ.copy()
+    env.update(fw_cfg.get("extra_env", {}))
+
+    # perf_dump_path for SGLang server-side timing (passed in request, zero overhead when None)
+    perf_dump_path = None
+    if framework == "sglang":
+        perf_dump_path = os.path.join(str(log_dir), f"perf_{case['id']}_measured.json")
+
+    log_file = log_dir / f"{case['id']}_{framework}.log"
+    log_fh = open(log_file, "w", encoding="utf-8", buffering=1)
+    log_thread = None
+
+    proc = None
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env,
+            preexec_fn=os.setsid,
+            text=True,
+            bufsize=1,
+        )
+
+        # Tee server output to both log file and stdout (like test_server_utils)
+        def _log_pipe(pipe, fh):
+            try:
+                for line in iter(pipe.readline, ""):
+                    sys.stdout.write(f"  [server] {line}")
+                    sys.stdout.flush()
+                    fh.write(line)
+            except ValueError:
+                pass  # pipe closed
+
+        log_thread = threading.Thread(target=_log_pipe, args=(proc.stdout, log_fh))
+        log_thread.daemon = True
+        log_thread.start()
+
+        base_url = f"http://{DEFAULT_HOST}:{port}"
+        wait_for_health(base_url, framework)
+
+        # Warmup requests (not measured, no perf dump)
+        # Use few steps to be fast — server's own warmup (warmup_steps=3) handles
+        # torch.compile compilation; these external warmups just stabilize triton
+        # kernel specializations across requests.
+        WARMUP_STEPS = 3
+        warmup_case = {**case, "num_inference_steps": WARMUP_STEPS}
+        for wi in range(1, 3):
+            print(f"  Sending warmup request ({wi}/2, {WARMUP_STEPS} steps)...")
+            try:
+                send_request(base_url, warmup_case, framework, config)
+            except Exception as e:
+                print(f"  Warmup request {wi} failed (non-fatal): {e}")
+
+        # Measured request — pass perf_dump_path for SGLang server-side timing
+        if perf_dump_path and os.path.exists(perf_dump_path):
+            os.remove(perf_dump_path)
+        print("  Sending measured request...")
+        latency = send_request(
+            base_url, case, framework, config, perf_dump_path=perf_dump_path
+        )
+        result["latency_s"] = round(latency, 3)
+
+    except Exception as e:
+        result["error"] = str(e)
+        print(f"  ERROR: {e}")
+    finally:
+        if proc:
+            kill_server(proc)
+        if log_thread:
+            log_thread.join(timeout=5)
+        log_fh.close()
+
+    return result
+
+
+def _install_framework(fw_name: str, dry_run: bool = False) -> bool:
+    """Install a comparison framework via the install script. Returns True on success."""
+    if fw_name not in INSTALLABLE_FRAMEWORKS:
+        return True
+    if not INSTALL_SCRIPT.exists():
+        print(f"  WARNING: Install script not found at {INSTALL_SCRIPT}")
+        return False
+    if dry_run:
+        print(f"  [DRY-RUN] Would install: bash {INSTALL_SCRIPT} {fw_name}")
+        return True
+    print(f"\n{'='*60}")
+    print(f"Installing framework: {fw_name}")
+    print(f"{'='*60}")
+    ret = subprocess.run(
+        ["bash", str(INSTALL_SCRIPT), fw_name],
+        timeout=600,
+    )
+    if ret.returncode != 0:
+        print(f"  WARNING: {fw_name} installation failed (exit {ret.returncode})")
+        return False
+    return True
+
+
+def run_comparison(
+    config: dict,
+    case_ids: list[str] | None = None,
+    frameworks: list[str] | None = None,
+    port: int = DEFAULT_PORT,
+    output: str = "comparison-results.json",
+    dry_run: bool = False,
+) -> dict:
+    """Run all comparison cases, grouped by framework to minimize installs.
+
+    Order: sglang first (already installed), then vllm-omni, then lightx2v.
+    Each non-sglang framework is installed right before its cases run.
+    """
+    timestamp = datetime.now(timezone.utc).isoformat()
+    commit_sha = os.environ.get("GITHUB_SHA", "unknown")
+    run_id = os.environ.get("GITHUB_RUN_ID", "local")
+
+    log_dir = Path("comparison-logs")
+    log_dir.mkdir(exist_ok=True)
+
+    # Collect all (case, framework) pairs, grouped by framework
+    fw_order = ["sglang", "vllm-omni", "lightx2v"]
+    fw_cases: dict[str, list[tuple[dict, dict]]] = {fw: [] for fw in fw_order}
+
+    for case in config["cases"]:
+        if case_ids and case["id"] not in case_ids:
+            continue
+        for fw_name, fw_cfg in case["frameworks"].items():
+            if frameworks and fw_name not in frameworks:
+                continue
+            if fw_name not in fw_cases:
+                fw_cases[fw_name] = []
+            fw_cases[fw_name].append((case, fw_cfg))
+
+    results = []
+    installed_fws: set[str] = set()
+
+    for fw_name in fw_order:
+        pairs = fw_cases.get(fw_name, [])
+        if not pairs:
+            continue
+
+        # Install framework if needed (once per framework)
+        if fw_name not in installed_fws and fw_name in INSTALLABLE_FRAMEWORKS:
+            if not _install_framework(fw_name, dry_run):
+                # Skip all cases for this framework
+                for case, _ in pairs:
+                    results.append(
+                        {
+                            "case_id": case["id"],
+                            "framework": fw_name,
+                            "model": case["model"],
+                            "task": case["task"],
+                            "latency_s": None,
+                            "error": f"{fw_name} installation failed",
+                        }
+                    )
+                continue
+            installed_fws.add(fw_name)
+
+        for case, fw_cfg in pairs:
+            print(f"\n{'='*60}")
+            print(f"Case: {case['id']} | Model: {case['model']} | Framework: {fw_name}")
+            print(f"{'='*60}")
+
+            if dry_run:
+                cmd = build_server_cmd(fw_name, case, fw_cfg, port)
+                print(f"  [DRY-RUN] Would run: {' '.join(cmd)}")
+                results.append(
+                    {
+                        "case_id": case["id"],
+                        "framework": fw_name,
+                        "model": case["model"],
+                        "task": case["task"],
+                        "latency_s": None,
+                        "error": "dry-run",
+                    }
+                )
+                continue
+
+            result = run_single(case, fw_name, fw_cfg, port, log_dir, config)
+            results.append(result)
+
+            # Wait for GPU memory to clear
+            print(f"  Waiting {GPU_CLEAR_WAIT}s for GPU memory to clear...")
+            time.sleep(GPU_CLEAR_WAIT)
+
+    output_data = {
+        "timestamp": timestamp,
+        "commit_sha": commit_sha,
+        "run_id": run_id,
+        "results": results,
+    }
+
+    os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
+    with open(output, "w") as f:
+        json.dump(output_data, f, indent=2)
+    print(f"\nResults written to {output}")
+
+    # Print summary table
+    print(f"\n{'='*60}")
+    print("SUMMARY")
+    print(f"{'='*60}")
+    for r in results:
+        lat = f"{r['latency_s']:.2f}s" if r["latency_s"] else r.get("error", "N/A")
+        print(f"  {r['case_id']:30s} | {r['framework']:12s} | {lat}")
+
+    return output_data
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Cross-framework diffusion serving comparison benchmark"
+    )
+    parser.add_argument(
+        "--config",
+        default=str(CONFIGS_PATH),
+        help="Path to comparison_configs.json",
+    )
+    parser.add_argument(
+        "--case-ids",
+        nargs="+",
+        default=None,
+        help="Only run specific case IDs",
+    )
+    parser.add_argument(
+        "--frameworks",
+        nargs="+",
+        default=None,
+        help="Only run specific frameworks (sglang, vllm-omni, lightx2v)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=DEFAULT_PORT,
+        help="Server port",
+    )
+    parser.add_argument(
+        "--output",
+        default="comparison-results.json",
+        help="Output JSON path",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Parse config and print commands without launching servers",
+    )
+
+    args = parser.parse_args()
+
+    with open(args.config) as f:
+        config = json.load(f)
+
+    print(f"Loaded {len(config['cases'])} comparison case(s) from {args.config}")
+
+    run_comparison(
+        config=config,
+        case_ids=args.case_ids,
+        frameworks=args.frameworks,
+        port=args.port,
+        output=args.output,
+        dry_run=args.dry_run,
+    )
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/diffusion/save_diffusion_metrics.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Collect and save diffusion performance metrics for artifact collection in CI.
+
+This script reads diffusion test results from the pytest stash and saves them
+with metadata for the performance dashboard.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/save_diffusion_metrics.py \
+        --gpu-config 1-gpu-h100 \
+        --run-id 12345678 \
+        --output test/diffusion-metrics-1gpu.json \
+        --results-json test/diffusion-results.json
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def load_diffusion_results(results_file: str) -> list[dict]:
+    """Load diffusion performance results from JSON file."""
+    if not os.path.exists(results_file):
+        print(f"Warning: Results file not found: {results_file}")
+        return []
+
+    try:
+        with open(results_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, list) else [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {results_file}: {e}")
+        return []
+
+
+def transform_diffusion_result(result: dict, gpu_config: str) -> dict:
+    """Transform a diffusion result to match dashboard expectations.
+
+    Dashboard expects:
+    - Separate test_name, class_name
+    - Numeric metrics in consistent units
+    - Optional modality field
+    """
+    return {
+        "test_name": result.get("test_name"),
+        "class_name": result.get("class_name"),
+        "modality": result.get("modality", "image"),
+        "e2e_ms": result.get("e2e_ms"),
+        "avg_denoise_ms": result.get("avg_denoise_ms"),
+        "median_denoise_ms": result.get("median_denoise_ms"),
+        "stage_metrics": result.get("stage_metrics", {}),
+        "sampled_steps": result.get("sampled_steps", {}),
+        # Video-specific metrics (if present)
+        "frames_per_second": result.get("frames_per_second"),
+        "total_frames": result.get("total_frames"),
+        "avg_frame_time_ms": result.get("avg_frame_time_ms"),
+    }
+
+
+def group_results_by_class(results: list[dict], gpu_config: str) -> list[dict]:
+    """Group diffusion results by test class (suite).
+
+    Returns list with one entry per test class, containing all tests in that class.
+    """
+    groups = {}
+
+    for result in results:
+        class_name = result.get("class_name", "unknown")
+
+        if class_name not in groups:
+            groups[class_name] = {
+                "gpu_config": gpu_config,
+                "test_suite": class_name,
+                "tests": [],
+            }
+
+        transformed = transform_diffusion_result(result, gpu_config)
+        groups[class_name]["tests"].append(transformed)
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    run_id: str,
+    output_file: str,
+    results_file: str,
+) -> bool:
+    """Collect diffusion metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Load diffusion results
+    raw_results = load_diffusion_results(results_file)
+    print(f"Loaded {len(raw_results)} diffusion test result(s)")
+
+    # Group by test class
+    grouped = group_results_by_class(raw_results, gpu_config)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "test_type": "diffusion",
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not raw_results:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved diffusion metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect diffusion performance metrics from test results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 1-gpu-h100, 2-gpu-h100)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--results-json",
+        required=True,
+        help="Path to diffusion results JSON file",
+    )
+
+    args = parser.parse_args()
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        run_id=args.run_id,
+        output_file=args.output,
+        results_file=args.results_json,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/install_protoc.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Ensure protoc is installed for router build (gRPC protobuf compilation).
+set -euxo pipefail
+
+if command -v protoc >/dev/null 2>&1 && protoc --version >/dev/null 2>&1; then
+    echo "protoc already installed: $(protoc --version)"
+    exit 0
+fi
+
+if command -v protoc >/dev/null 2>&1; then
+    echo "protoc found but not runnable, reinstalling..."
+else
+    echo "protoc not found, installing..."
+fi
+
+ARCH=$(uname -m)
+
+if command -v apt-get &> /dev/null; then
+    # Ubuntu/Debian
+    apt-get update || true # May fail due to unrelated broken packages
+    PROTOC_APT_PACKAGES=(wget unzip gcc g++ perl make)
+    apt-get install -y --no-install-recommends "${PROTOC_APT_PACKAGES[@]}" || {
+        echo "Warning: apt-get install failed, checking if required packages are available..."
+        for pkg in "${PROTOC_APT_PACKAGES[@]}"; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages are already installed, continuing..."
+    }
+elif command -v yum &> /dev/null; then
+    # RHEL/CentOS
+    yum update -y
+    yum install -y wget unzip gcc gcc-c++ perl-core make
+else
+    echo "ERROR: Neither apt-get nor yum found; cannot install protoc build deps"
+    exit 1
+fi
+
+if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+    PROTOC_ARCH="aarch_64"
+else
+    PROTOC_ARCH="x86_64"
+fi
+PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip"
+(
+    cd /tmp
+    wget "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP}"
+    unzip -o "${PROTOC_ZIP}" -d /usr/local
+    rm -f "${PROTOC_ZIP}"
+)
+protoc --version

third_party/sglang/scripts/ci/utils/merge_metrics.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Merge per-partition metrics into a consolidated metrics file.
+
+This script reads all per-partition metric JSON files and consolidates them
+into a single JSON file with run-level metadata.
+
+Usage:
+    python3 scripts/ci/utils/merge_metrics.py \
+        --input-dir metrics/ \
+        --output consolidated-metrics-12345678.json \
+        --run-id 12345678 \
+        --commit-sha abc123def456
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_partition_files(input_dir: str) -> list[str]:
+    """Find all partition metric files in the input directory."""
+    patterns = [
+        os.path.join(input_dir, "**/metrics-*.json"),
+        os.path.join(input_dir, "**/diffusion-metrics-*.json"),
+        os.path.join(input_dir, "**/comparison-metrics-*.json"),
+    ]
+    files = set()
+    for pattern in patterns:
+        files.update(glob.glob(pattern, recursive=True))
+    return list(files)
+
+
+def load_partition_metrics(filepath: str) -> dict | None:
+    """Load a partition metrics file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to load {filepath}: {e}")
+        return None
+
+
+def merge_metrics(
+    input_dir: str,
+    output_file: str,
+    run_id: str,
+    commit_sha: str,
+    branch: str | None = None,
+) -> bool:
+    """Merge all partition metrics into a consolidated file."""
+    run_date = datetime.now(timezone.utc).isoformat()
+
+    # Find all partition files
+    partition_files = find_partition_files(input_dir)
+    print(f"Found {len(partition_files)} partition file(s)")
+
+    all_results = []
+    if not partition_files:
+        print("No partition metrics files found")
+    else:
+        # Load all partition files
+        for filepath in sorted(partition_files):
+            print(f"  Reading: {filepath}")
+            metrics = load_partition_metrics(filepath)
+            if metrics and "results" in metrics:
+                all_results.extend(metrics["results"])
+        print(f"Total results collected: {len(all_results)}")
+
+    # Create consolidated structure
+    consolidated = {
+        "run_id": run_id,
+        "run_date": run_date,
+        "commit_sha": commit_sha,
+        "branch": branch,
+        "results": all_results,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(consolidated, f, indent=2)
+
+        if not partition_files:
+            print(f"Created empty consolidated file: {output_file}")
+        else:
+            print(f"Saved consolidated metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing consolidated file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Merge per-partition metrics into consolidated file"
+    )
+    parser.add_argument(
+        "--input-dir",
+        required=True,
+        help="Directory containing partition metric files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for consolidated metrics JSON",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--commit-sha",
+        required=True,
+        help="Git commit SHA",
+    )
+    parser.add_argument(
+        "--branch",
+        default=None,
+        help="Git branch name (optional)",
+    )
+
+    args = parser.parse_args()
+
+    success = merge_metrics(
+        input_dir=args.input_dir,
+        output_file=args.output,
+        run_id=args.run_id,
+        commit_sha=args.commit_sha,
+        branch=args.branch,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/prevalidate_cached_models.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+"""
+Pre-validate all cached HuggingFace models to provide detailed feedback.
+
+This script runs once during CI initialization (in prepare_runner.sh) to:
+1. Scan snapshots in ~/.cache/huggingface/hub/ (with time/quantity limits)
+2. Validate completeness (config/tokenizer/weights)
+3. Output detailed failure reasons for debugging
+
+NOTE: This script no longer writes shared validation markers. Each test run
+independently validates its cache using per-run markers to avoid cross-runner
+cache state pollution.
+"""
+
+import glob
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Add python directory to path to import sglang modules
+REPO_ROOT = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(REPO_ROOT / "python"))
+
+from sglang.srt.model_loader.ci_weight_validation import (  # noqa: E402
+    _validate_diffusion_model,
+    validate_cache_with_detailed_reason,
+)
+
+# Limits to avoid spending too much time on validation
+MAX_VALIDATION_TIME_SECONDS = 300  # Max 5 minutes total
+
+
+def find_all_hf_snapshots():
+    """
+    Find all HuggingFace snapshots in cache.
+
+    Returns:
+        List of (model_name, snapshot_dir) tuples, sorted by mtime (newest first)
+    """
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    hub_dir = os.path.join(hf_home, "hub")
+
+    if not os.path.isdir(hub_dir):
+        print(f"HF hub directory not found: {hub_dir}")
+        return []
+
+    snapshots = []
+
+    # Pattern: models--org--model/snapshots/hash
+    for model_dir in glob.glob(os.path.join(hub_dir, "models--*")):
+        # Extract model name from directory (models--org--model -> org/model)
+        dir_name = os.path.basename(model_dir)
+        if not dir_name.startswith("models--"):
+            continue
+
+        # models--meta-llama--Llama-2-7b-hf -> meta-llama/Llama-2-7b-hf
+        # Handle multi-part names: models--a--b--c -> a/b-c (join parts 1+ with /)
+        parts = dir_name.split("--")
+        if len(parts) < 3 or parts[0] != "models":
+            # Invalid format, skip
+            continue
+        # Standard format: models--org--repo -> org/repo
+        # Extended format: models--org--repo--extra -> org/repo-extra (join with -)
+        model_name = parts[1] + "/" + "-".join(parts[2:])
+
+        snapshots_dir = os.path.join(model_dir, "snapshots")
+        if not os.path.isdir(snapshots_dir):
+            continue
+
+        # Find all snapshot hashes
+        for snapshot_hash_dir in os.listdir(snapshots_dir):
+            snapshot_path = os.path.join(snapshots_dir, snapshot_hash_dir)
+            if os.path.isdir(snapshot_path):
+                try:
+                    mtime = os.path.getmtime(snapshot_path)
+                    snapshots.append((model_name, snapshot_path, mtime))
+                except OSError:
+                    continue
+
+    # Sort by mtime (newest first) - prioritize recently used models
+    snapshots.sort(key=lambda x: x[2], reverse=True)
+
+    # Return without mtime
+    return [(name, path) for name, path, _ in snapshots]
+
+
+def is_transformers_text_model(snapshot_dir):
+    """
+    Check if a snapshot is a transformers text model.
+
+    Only excludes (returns False) for models with STRONG evidence of being
+    diffusers/generation pipelines. Uses conservative heuristics to avoid
+    false negatives on multimodal LLMs with tokenizers.
+
+    Args:
+        snapshot_dir: Path to snapshot directory
+
+    Returns:
+        True if this looks like a transformers text model, False otherwise (N/A)
+    """
+    # Check for diffusers pipeline markers (strong evidence)
+    diffusers_markers = [
+        "model_index.json",  # Diffusers pipeline config
+        "scheduler",  # Scheduler directory (diffusers)
+    ]
+    if any(
+        os.path.exists(os.path.join(snapshot_dir, marker))
+        for marker in diffusers_markers
+    ):
+        return False
+
+    config_path = os.path.join(snapshot_dir, "config.json")
+    if not os.path.exists(config_path):
+        # No config.json - likely not a transformers model
+        return False
+
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+
+        # Check for explicit diffusers/generation model types (conservative keywords)
+        model_type = config.get("_class_name") or config.get("model_type")
+        if model_type:
+            model_type_lower = str(model_type).lower()
+            # Only exclude clear diffusion/generation models
+            if any(
+                keyword in model_type_lower
+                for keyword in [
+                    "diffusion",
+                    "unet",
+                    "vae",
+                    "controlnet",
+                    "stable-diffusion",
+                    "latent-diffusion",
+                ]
+            ):
+                return False
+
+        # Check architectures for explicit generation/diffusion classes
+        architectures = config.get("architectures", [])
+        if architectures:
+            arch_str = " ".join(architectures).lower()
+            # Conservative: only exclude obvious diffusion/generation architectures
+            # Use word boundaries to avoid false positives (e.g., "dit" in "conditional")
+            for keyword in [
+                "diffusion",
+                "unet2d",
+                "unet3d",
+                "vaedecoder",  # More specific than "vae"
+                "vaeencoder",
+                "controlnet",
+                "autoencoder",
+                "ditmodel",  # Diffusion Transformer - use more specific pattern
+                "pixart",  # PixArt diffusion model
+            ]:
+                if keyword in arch_str:
+                    return False
+
+        # Check for standalone vision encoder/image processor (no text component)
+        # Only if model name explicitly indicates non-text usage
+        model_name = config.get("_name_or_path", "").lower()
+
+        if any(
+            keyword in model_name
+            for keyword in [
+                "image-edit-",  # Pure image editing (e.g., Qwen-Image-Edit)
+                "-image-editing",
+                "dit-",  # DiT generation models
+                "pixart-",  # PixArt generation models
+            ]
+        ):
+            # Additional check: does it have tokenizer? If yes, might be multimodal LLM
+            has_tokenizer = any(
+                os.path.exists(os.path.join(snapshot_dir, fname))
+                for fname in ["tokenizer.json", "tokenizer.model", "tiktoken.model"]
+            )
+            if not has_tokenizer:
+                # Image-edit model without tokenizer -> likely pure vision pipeline
+                return False
+
+        # Default: assume it's a transformers text/multimodal model
+        # Even if it lacks tokenizer, let validation report the actual error
+        # (better false positive than false negative for text models)
+        return True
+
+    except (json.JSONDecodeError, OSError, KeyError):
+        # Can't parse config - assume it's transformers and let validation report failure
+        return True
+
+
+def scan_weight_files(snapshot_dir):
+    """
+    Scan for weight files in a snapshot.
+
+    Returns:
+        List of weight file paths, or empty list if scan fails
+    """
+    weight_files = []
+
+    # First, look for index files
+    index_patterns = ["*.safetensors.index.json", "pytorch_model.bin.index.json"]
+    index_files = []
+    for pattern in index_patterns:
+        index_files.extend(glob.glob(os.path.join(snapshot_dir, pattern)))
+
+    # If we have safetensors index, collect shards from it
+    for index_file in index_files:
+        if index_file.endswith(".safetensors.index.json"):
+            try:
+                with open(index_file, "r", encoding="utf-8") as f:
+                    index_data = json.load(f)
+                weight_map = index_data.get("weight_map", {})
+                for weight_file in set(weight_map.values()):
+                    weight_path = os.path.join(snapshot_dir, weight_file)
+                    if os.path.exists(weight_path):
+                        weight_files.append(weight_path)
+            except Exception as e:
+                print(
+                    f"  Warning: Failed to parse index {os.path.basename(index_file)}: {e}"
+                )
+
+    # If no index found or no shards from index, do recursive glob
+    if not weight_files:
+        matched = glob.glob(
+            os.path.join(snapshot_dir, "**/*.safetensors"), recursive=True
+        )
+        MAX_WEIGHT_FILES = 1000
+        if len(matched) > MAX_WEIGHT_FILES:
+            print(
+                f"  Warning: Too many safetensors files ({len(matched)} > {MAX_WEIGHT_FILES})"
+            )
+            return []
+
+        for f in matched:
+            if os.path.exists(f):  # Filter out broken symlinks
+                weight_files.append(f)
+
+    return weight_files
+
+
+def validate_snapshot(model_name, snapshot_dir, weight_files, validated_cache):
+    """
+    Validate a snapshot and return detailed status.
+
+    Uses in-process cache to avoid duplicate validation within the same run.
+
+    Args:
+        model_name: Model identifier
+        snapshot_dir: Path to snapshot directory
+        weight_files: List of weight files to validate
+        validated_cache: Dict to track already-validated snapshots in this run
+
+    Returns:
+        Tuple of (result, reason):
+        - (True, None) if validation passed
+        - (False, reason_str) if validation failed
+        - (None, None) if skipped (already validated in this run)
+    """
+    # Fast path: check in-process cache first
+    if snapshot_dir in validated_cache:
+        return None, None  # Already validated in this run, skip
+
+    try:
+        # Perform validation with detailed reason
+        is_complete, reason = validate_cache_with_detailed_reason(
+            snapshot_dir=snapshot_dir,
+            weight_files=weight_files,
+            model_name_or_path=model_name,
+        )
+
+        # Cache result to avoid re-validation in this run
+        validated_cache[snapshot_dir] = (is_complete, reason)
+
+        return is_complete, reason
+
+    except Exception as e:
+        error_msg = f"Validation raised exception: {e}"
+        return False, error_msg
+
+
+def main():
+    start_time = time.time()
+
+    print("=" * 70)
+    print("CI_OFFLINE: Pre-validating cached HuggingFace models")
+    print("=" * 70)
+    print(f"Max time: {MAX_VALIDATION_TIME_SECONDS}s")
+    print()
+
+    print("Scanning HuggingFace cache for models...")
+    snapshots = find_all_hf_snapshots()
+
+    if not snapshots:
+        print("No cached models found, skipping validation")
+        print("=" * 70)
+        return
+
+    print(f"Found {len(snapshots)} snapshot(s) in cache")
+    print()
+
+    validated_count = 0
+    failed_count = 0
+    skipped_count = 0
+    processed_count = 0
+
+    # In-process cache to avoid re-validating same snapshot in this run
+    validated_cache = {}
+
+    for model_name, snapshot_dir in snapshots:
+        # Check time limit
+        elapsed = time.time() - start_time
+        if elapsed > MAX_VALIDATION_TIME_SECONDS:
+            print()
+            print(
+                f"Time limit reached ({elapsed:.1f}s > {MAX_VALIDATION_TIME_SECONDS}s)"
+            )
+            print(
+                f"Stopping validation, {len(snapshots) - processed_count} snapshots remaining"
+            )
+            break
+
+        snapshot_hash = os.path.basename(snapshot_dir)
+        print(
+            f"[{processed_count + 1}/{len(snapshots)}] {model_name} ({snapshot_hash[:8]}...)"
+        )
+        processed_count += 1
+
+        # Determine model type by checking for model_index.json (diffusers pipeline marker)
+        model_index_path = os.path.join(snapshot_dir, "model_index.json")
+        is_diffusion_model = os.path.exists(model_index_path)
+
+        if is_diffusion_model:
+            # This is a diffusers pipeline - use diffusion validation
+            try:
+                is_valid, reason = _validate_diffusion_model(snapshot_dir)
+
+                if is_valid:
+                    print("  PASS (diffusion) - Cache complete & valid")
+                    validated_count += 1
+                else:
+                    print(f"  FAIL (diffusion) - {reason}")
+                    failed_count += 1
+
+            except Exception as e:
+                print(f"  FAIL (diffusion) - Validation raised exception: {e}")
+                failed_count += 1
+
+            continue
+
+        # Transformers model - use standard validation
+        # First check if this looks like a transformers text model
+        if not is_transformers_text_model(snapshot_dir):
+            # Not a recognized model type, skip
+            print(
+                "  SKIP (unknown type) - Not a diffusers pipeline or transformers model"
+            )
+            skipped_count += 1
+            continue
+
+        # Scan weight files
+        weight_files = scan_weight_files(snapshot_dir)
+
+        if not weight_files:
+            print("  SKIP (no weights) - empty or incomplete download")
+            skipped_count += 1
+            continue
+
+        # Validate
+        try:
+            result, reason = validate_snapshot(
+                model_name, snapshot_dir, weight_files, validated_cache
+            )
+
+            if result is True:
+                print("  PASS - Cache complete & valid")
+                validated_count += 1
+            elif result is False:
+                # Print detailed failure reason
+                if reason:
+                    print(f"  FAIL (incomplete) - {reason}")
+                else:
+                    print("  FAIL (incomplete) - cache validation failed")
+                failed_count += 1
+            else:  # None (skipped)
+                print("  SKIP (already validated in this run)")
+                skipped_count += 1
+
+        except Exception as e:
+            print(f"  FAIL (error) - Validation raised exception: {e}")
+            failed_count += 1
+
+    elapsed_total = time.time() - start_time
+
+    print()
+    print("=" * 70)
+    print(f"Validation summary (completed in {elapsed_total:.1f}s):")
+    print(f"  PASS (complete & valid):      {validated_count}")
+    print(f"  FAIL (incomplete/corrupted):  {failed_count}")
+    print(f"  SKIP (no weights/duplicate):  {skipped_count}")
+    print(f"  Total processed:              {processed_count}/{len(snapshots)}")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/publish_traces.py

@@ -0,0 +1,517 @@
+"""
+Publish performance traces to GitHub repository
+"""
+
+import argparse
+import base64
+import json
+import os
+import sys
+import time
+import warnings
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+
+def is_rate_limit_error(e):
+    """Check if an exception is a GitHub rate limit error (not permission error)"""
+    if not isinstance(e, HTTPError):
+        return False
+    if e.code == 429:
+        return True
+    if e.code == 403:
+        # 403 can be rate limit OR permission error - check the message
+        error_body = getattr(e, "error_body", "")
+        if isinstance(error_body, str):
+            # Rate limit errors contain specific phrases
+            rate_limit_phrases = [
+                "rate limit",
+                "abuse detection",
+                "secondary rate limit",
+            ]
+            return any(phrase in error_body.lower() for phrase in rate_limit_phrases)
+    return False
+
+
+def is_permission_error(e):
+    """Check if an exception is a GitHub permission error"""
+    if not isinstance(e, HTTPError) or e.code != 403:
+        return False
+    error_body = getattr(e, "error_body", "")
+    if isinstance(error_body, str):
+        permission_phrases = [
+            "resource not accessible",
+            "must have push access",
+            "permission",
+            "denied",
+        ]
+        return any(phrase in error_body.lower() for phrase in permission_phrases)
+    return False
+
+
+def make_github_request(url, token, method="GET", data=None):
+    """Make authenticated request to GitHub API"""
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        # "User-Agent": "sglang-ci",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+
+    if data:
+        headers["Content-Type"] = "application/json"
+        data = json.dumps(data).encode("utf-8")
+
+    req = Request(url, data=data, headers=headers, method=method)
+
+    try:
+        with urlopen(req) as response:
+            return response.read().decode("utf-8")
+    except HTTPError as e:
+        print(f"GitHub API request failed: {e}")
+        try:
+            error_body = e.read().decode("utf-8")
+            print(f"Error response body: {error_body}")
+            e.error_body = error_body  # Attach for later inspection
+        except Exception:
+            e.error_body = ""
+        raise
+    except Exception as e:
+        print(f"GitHub API request failed with a non-HTTP error: {e}")
+        raise
+
+
+def verify_token_permissions(repo_owner, repo_name, token):
+    """Verify that the token has necessary permissions for the repository"""
+    print("Verifying token permissions...")
+
+    checks = [
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}",  # Check if we can access the repository
+            "Repository access verified",
+        ),
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents",  # Check if we can read the repository contents
+            "Repository contents access verified",
+        ),
+    ]
+
+    for url, success_message in checks:
+        try:
+            response = make_github_request(url, token)
+            if success_message == "Repository access verified":
+                repo_data = json.loads(response)
+                print(f"{success_message}: {repo_data['full_name']}")
+            else:
+                print(success_message)
+        except Exception as e:
+            if is_rate_limit_error(e):
+                warnings.warn(
+                    "GitHub API rate limit exceeded during token verification."
+                )
+                return "rate_limited"
+            print(f"Failed to verify permissions for {url}: {e}")
+            return False
+
+    return True
+
+
+def get_branch_sha(repo_owner, repo_name, branch, token):
+    """Get SHA of the branch head"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["object"]["sha"]
+
+
+def get_tree_sha(repo_owner, repo_name, commit_sha, token):
+    """Get tree SHA from commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["tree"]["sha"]
+
+
+def create_blob(repo_owner, repo_name, content, token, max_retries=3):
+    """Create a blob with file content"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs"
+
+    # Encode content as base64 for GitHub API
+    content_b64 = base64.b64encode(content).decode("utf-8")
+
+    data = {"content": content_b64, "encoding": "base64"}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt  # Exponential backoff: 1s, 2s, 4s
+                print(
+                    f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_blobs(repo_owner, repo_name, files, token):
+    """Create blobs for all files and return tree items with blob SHAs"""
+    tree_items = []
+    for i, (file_path, content) in enumerate(files):
+        # Create blob first to get SHA
+        blob_sha = create_blob(repo_owner, repo_name, content, token)
+        tree_items.append(
+            {
+                "path": file_path,
+                "mode": "100644",
+                "type": "blob",
+                "sha": blob_sha,
+            }
+        )
+        # Progress indicator for large uploads
+        if (i + 1) % 10 == 0 or (i + 1) == len(files):
+            print(f"Created {i + 1}/{len(files)} blobs...")
+    return tree_items
+
+
+def create_tree(repo_owner, repo_name, base_tree_sha, tree_items, token, max_retries=3):
+    """Create a new tree from pre-created blob SHAs"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees"
+
+    data = {"base_tree": base_tree_sha, "tree": tree_items}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_commit(
+    repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3
+):
+    """Create a new commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits"
+
+    data = {"tree": tree_sha, "parents": [parent_sha], "message": message}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            commit_sha = json.loads(response)["sha"]
+
+            # Verify the commit was actually created
+            verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+            verify_response = make_github_request(verify_url, token)
+            verify_data = json.loads(verify_response)
+            if verify_data["sha"] != commit_sha:
+                raise Exception(
+                    f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}"
+                )
+
+            return commit_sha
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3):
+    """Update branch reference to point to new commit"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+
+    data = {"sha": commit_sha}
+
+    for attempt in range(max_retries):
+        try:
+            make_github_request(url, token, method="PATCH", data=data)
+            return
+        except HTTPError as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            # Check if this is an "Object does not exist" error
+            is_object_not_exist = False
+            if hasattr(e, "error_body"):
+                try:
+                    error_data = json.loads(e.error_body)
+                    if "Object does not exist" in error_data.get("message", ""):
+                        is_object_not_exist = True
+                except Exception:
+                    pass
+
+            if is_object_not_exist and attempt < max_retries - 1:
+                # This might be a transient consistency issue - wait and retry
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def copy_trace_files(source_dir, target_base_path):
+    """Copy trace files and return list of files to upload.
+
+    Only uploads traces from TP rank 0 to avoid duplicated data across tensor parallel ranks.
+    """
+    files_to_upload = []
+
+    if not os.path.exists(source_dir):
+        print(f"Warning: Traces directory {source_dir} does not exist")
+        return files_to_upload
+
+    # Walk through source directory and find .json.gz files
+    for root, dirs, files in os.walk(source_dir):
+        for file in files:
+            if file.endswith(".json.gz"):
+
+                # Only upload TP rank 0 traces to avoid duplicates across tensor parallel ranks
+                if "TP-" in file and "TP-0" not in file:
+                    continue
+
+                source_file = os.path.join(root, file)
+                # Calculate relative path from source_dir
+                rel_path = os.path.relpath(source_file, source_dir)
+                target_path = f"{target_base_path}/{rel_path}"
+
+                # Read file content
+                with open(source_file, "rb") as f:
+                    content = f.read()
+
+                files_to_upload.append((target_path, content))
+
+    return files_to_upload
+
+
+def publish_traces(traces_dir, run_id, run_number):
+    """Publish traces from a single directory to GitHub repository in a single commit"""
+    target_base_path = f"traces/{run_id}"
+    files_to_upload = copy_trace_files(traces_dir, target_base_path)
+
+    if not files_to_upload:
+        print("No trace files found to upload")
+        return
+
+    print(f"Found {len(files_to_upload)} files to upload")
+    publish_traces_from_files(files_to_upload, run_id, run_number)
+
+
+def publish_traces_from_files(files_to_upload, run_id, run_number):
+    """Publish pre-collected trace files to GitHub repository in a single commit"""
+    # Get environment variables
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    # Repository configuration
+    repo_owner = "sglang-bot"
+    repo_name = "sglang-ci-data"
+    branch = "main"
+
+    # Verify token permissions before proceeding
+    permission_check = verify_token_permissions(repo_owner, repo_name, token)
+    if permission_check == "rate_limited":
+        warnings.warn(
+            "Skipping trace upload due to GitHub API rate limit. "
+            "This is expected during high CI activity and does not indicate a test failure."
+        )
+        return
+    elif not permission_check:
+        print(
+            "Token permission verification failed. Please check the token permissions."
+        )
+        sys.exit(1)
+
+    max_retries = 5
+    retry_delay = 5  # seconds
+
+    # Create blobs once before retry loop to avoid re-uploading on failures
+    try:
+        tree_items = create_blobs(repo_owner, repo_name, files_to_upload, token)
+    except Exception as e:
+        # Check for rate limit errors during blob creation
+        if is_rate_limit_error(e):
+            warnings.warn(
+                "GitHub API rate limit exceeded during blob creation. Skipping trace upload."
+            )
+            return
+        # Check for permission errors - these should fail loudly
+        if is_permission_error(e):
+            print(
+                f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                "'contents: write' permission for the repository."
+            )
+            sys.exit(1)
+        print(f"Failed to create blobs: {e}")
+        raise
+
+    for attempt in range(max_retries):
+        try:
+            # Get current branch head
+            branch_sha = get_branch_sha(repo_owner, repo_name, branch, token)
+            print(f"Current branch head: {branch_sha}")
+
+            # Get current tree
+            tree_sha = get_tree_sha(repo_owner, repo_name, branch_sha, token)
+            print(f"Current tree SHA: {tree_sha}")
+
+            # Create new tree with pre-created blobs
+            new_tree_sha = create_tree(
+                repo_owner, repo_name, tree_sha, tree_items, token
+            )
+            print(f"Created new tree: {new_tree_sha}")
+
+            # Create commit
+            commit_message = f"Nightly traces for run {run_id} at {run_number} ({len(files_to_upload)} files)"
+            commit_sha = create_commit(
+                repo_owner,
+                repo_name,
+                new_tree_sha,
+                branch_sha,
+                commit_message,
+                token,
+            )
+            print(f"Created commit: {commit_sha}")
+
+            # Update branch reference
+            update_branch_ref(repo_owner, repo_name, branch, commit_sha, token)
+            print("Updated branch reference")
+
+            print("Successfully published all traces in a single commit")
+            return
+
+        except Exception as e:
+            # Check for retryable errors
+            is_retryable = False
+            error_type = "unknown"
+
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    is_retryable = True
+                    error_type = "fast-forward conflict"
+                elif "Object does not exist" in e.error_body:
+                    is_retryable = True
+                    error_type = "object consistency"
+
+            # Also retry on HTTP errors that might be transient
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+                error_type = f"HTTP {e.code}"
+
+            # Check for rate limit errors (non-fatal - just warn and skip)
+            if is_rate_limit_error(e):
+                warnings.warn("GitHub API rate limit exceeded. Skipping trace upload.")
+                return
+
+            # Check for permission errors - these should fail loudly
+            if is_permission_error(e):
+                print(
+                    f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                    "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                    "'contents: write' permission for the repository."
+                )
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish traces after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish performance traces to GitHub repository"
+    )
+    parser.add_argument(
+        "--traces-dir",
+        type=str,
+        action="append",
+        dest="traces_dirs",
+        required=True,
+        help="Traces directory to publish (can be specified multiple times)",
+    )
+    args = parser.parse_args()
+
+    # Get environment variables
+    run_id = os.getenv("GITHUB_RUN_ID", "test")
+    run_number = os.getenv("GITHUB_RUN_NUMBER", "12345")
+
+    if not run_id or not run_number:
+        print(
+            "Error: GITHUB_RUN_ID and GITHUB_RUN_NUMBER environment variables must be set"
+        )
+        sys.exit(1)
+
+    # Collect trace files from all directories
+    target_base_path = f"traces/{run_id}"
+    all_files = []
+    for traces_dir in args.traces_dirs:
+        print(f"Processing traces from directory: {traces_dir}")
+        files = copy_trace_files(traces_dir, target_base_path)
+        all_files.extend(files)
+
+    if not all_files:
+        print("No trace files found to upload across all directories")
+        return
+
+    print(f"Found {len(all_files)} total files to upload")
+
+    # Publish all collected traces in a single commit
+    publish_traces_from_files(all_files, run_id, run_number)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/runner_utilization_report.py

@@ -0,0 +1,527 @@
+#!/usr/bin/env python3
+"""
+Runner Utilization Report
+
+Analyzes GitHub Actions job data to calculate runner utilization metrics.
+Reports idle time, active time, and utilization percentage per runner label.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta, timezone
+
+# Labels to skip when grouping runners (GitHub default labels)
+DEFAULT_LABELS_TO_IGNORE = {"self-hosted", "Linux", "X64", "ARM64"}
+GITHUB_HOSTED_LABELS = {"ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"}
+
+
+def run_gh_command(args: list[str]) -> dict:
+    """Run gh CLI command and return JSON result."""
+    result = subprocess.run(
+        ["gh", "api"] + args,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise Exception(f"gh api failed: {result.stderr}")
+    return json.loads(result.stdout)
+
+
+def get_workflow_runs(repo: str, hours: int = 24) -> list[dict]:
+    """Get workflow runs from the last N hours."""
+    since = datetime.now(timezone.utc) - timedelta(hours=hours)
+
+    runs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs?per_page=100&page={page}",
+            ]
+        )
+        page_runs = data.get("workflow_runs", [])
+
+        # Filter by time
+        for run in page_runs:
+            created_at = parse_time(run.get("created_at"))
+            if created_at and created_at >= since:
+                runs.append(run)
+            elif created_at and created_at < since:
+                # Runs are ordered by created_at desc, so we can stop
+                return runs
+
+        if len(page_runs) < 100:
+            break
+        page += 1
+        if page > 20:  # Safety limit
+            break
+    return runs
+
+
+def get_jobs_for_run(repo: str, run_id: int) -> list[dict]:
+    """Get all jobs for a workflow run."""
+    jobs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}",
+            ]
+        )
+        jobs.extend(data.get("jobs", []))
+        if len(data.get("jobs", [])) < 100:
+            break
+        page += 1
+        if page > 5:  # Safety limit
+            break
+    return jobs
+
+
+def get_runners(repo: str, online_only: bool = True) -> list[dict]:
+    """Get all self-hosted runners with pagination. Returns empty if no permission."""
+    try:
+        all_runners = []
+        page = 1
+        while True:
+            data = run_gh_command(
+                [f"repos/{repo}/actions/runners?per_page=100&page={page}"]
+            )
+            runners = data.get("runners", [])
+            all_runners.extend(runners)
+            if len(runners) < 100:
+                break
+            page += 1
+            if page > 10:  # Safety limit
+                break
+        if online_only:
+            all_runners = [r for r in all_runners if r.get("status") == "online"]
+        return all_runners
+    except Exception as e:
+        print(f"Warning: Cannot access runners API (need admin): {e}")
+        return []
+
+
+def parse_time(time_str: str) -> datetime:
+    """Parse ISO timestamp to datetime."""
+    if not time_str:
+        return None
+    return datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+
+
+# Known runner counts per label (fallback when API unavailable)
+KNOWN_RUNNER_COUNTS = {
+    "1-gpu-5090": 16,
+    "h200": 8,
+    "h20": 4,
+    "b200": 4,
+    "amd": 8,
+    "github-hosted": 20,  # GitHub hosted runners (variable)
+    "other": 10,
+}
+
+
+def calculate_concurrency_metrics(
+    jobs: list[dict],
+    window_start: datetime,
+    window_end: datetime,
+    num_runners: int,
+) -> dict:
+    """
+    Calculate concurrency metrics using a sweep line algorithm.
+
+    Tracks:
+    - Peak concurrent runners in use
+    - Average concurrent runners over time
+    - Time at saturation (all runners busy)
+    - Queue depth when runners are saturated
+    """
+    if not jobs:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+
+    window_seconds = (window_end - window_start).total_seconds()
+    if window_seconds <= 0:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+
+    # Create events for running jobs: +1 at start, -1 at end
+    running_events = []
+    for job in jobs:
+        start = job["start"]
+        end = job["end"]
+        # Clamp to window
+        if end < window_start or start > window_end:
+            continue
+        clamped_start = max(start, window_start)
+        clamped_end = min(end, window_end)
+        running_events.append((clamped_start, 1, "start"))  # +1 for start
+        running_events.append((clamped_end, -1, "end"))  # -1 for end
+
+    # Create events for queue tracking (jobs created but not started)
+    queue_events = []
+    for job in jobs:
+        created_at = job.get("created_at")
+        started_at = job["start"]
+        if created_at and created_at < started_at:
+            # Clamp to window
+            if started_at < window_start or created_at > window_end:
+                continue
+            clamped_created = max(created_at, window_start)
+            clamped_started = min(started_at, window_end)
+            queue_events.append((clamped_created, 1, "queued"))
+            queue_events.append((clamped_started, -1, "dequeued"))
+
+    # Sort running events: by time, then ends before starts at same time
+    running_events.sort(key=lambda e: (e[0], e[1] == 1))
+
+    # Process running events to get concurrency metrics
+    current_running = 0
+    peak_running = 0
+    prev_time = window_start
+    total_running_seconds = 0.0
+    saturation_seconds = 0.0
+
+    for event_time, delta, _ in running_events:
+        # Accumulate time at previous concurrency level
+        time_delta = (event_time - prev_time).total_seconds()
+        if time_delta > 0:
+            total_running_seconds += current_running * time_delta
+            if current_running >= num_runners:
+                saturation_seconds += time_delta
+
+        # Update concurrency
+        current_running += delta
+        peak_running = max(peak_running, current_running)
+        prev_time = event_time
+
+    # Handle remaining time after last event
+    if prev_time < window_end:
+        time_delta = (window_end - prev_time).total_seconds()
+        total_running_seconds += current_running * time_delta
+        if current_running >= num_runners:
+            saturation_seconds += time_delta
+
+    # Sort queue events and calculate peak queue depth
+    queue_events.sort(key=lambda e: (e[0], e[1] == 1))
+    current_queued = 0
+    peak_queue = 0
+
+    for _, delta, _ in queue_events:
+        current_queued += delta
+        peak_queue = max(peak_queue, current_queued)
+
+    avg_concurrent = total_running_seconds / window_seconds if window_seconds > 0 else 0
+
+    return {
+        "peak_concurrent": peak_running,
+        "avg_concurrent": avg_concurrent,
+        "saturation_seconds": saturation_seconds,
+        "saturation_pct": (
+            (saturation_seconds / window_seconds * 100) if window_seconds > 0 else 0
+        ),
+        "peak_queue": peak_queue,
+    }
+
+
+def calculate_utilization(repo: str, hours: int = 24, runner_filter: str = None):
+    """Calculate runner utilization metrics."""
+
+    print(f"Fetching workflow runs from last {hours} hours...")
+    runs = get_workflow_runs(repo, hours)
+    print(f"Found {len(runs)} workflow runs")
+
+    # Try to get online runners from API
+    print("Fetching online runners...")
+    runners = get_runners(repo, online_only=True)
+
+    # Build label -> set of online runner names from API
+    api_label_runners = defaultdict(set)
+    if runners:
+        for runner in runners:
+            for label in runner.get("labels", []):
+                label_name = label.get("name", "")
+                if label_name not in DEFAULT_LABELS_TO_IGNORE:
+                    api_label_runners[label_name].add(runner["name"])
+        print(f"Got {len(runners)} online runners from API")
+    else:
+        print("No runner API access, will use observed runners from job data")
+
+    # Track runners seen in jobs (for labels not in API or when API unavailable)
+    job_label_runners = defaultdict(set)
+    label_jobs = defaultdict(list)  # label -> list of job_info
+
+    # Fetch jobs for all runs in parallel
+    total_runs = len(runs)
+    print(f"Fetching jobs for {total_runs} runs in parallel...")
+
+    def fetch_jobs_for_run(run):
+        """Fetch jobs for a single run, returning (run_id, jobs) or (run_id, None) on error."""
+        try:
+            return (run["id"], get_jobs_for_run(repo, run["id"]))
+        except Exception:
+            return (run["id"], None)
+
+    all_jobs = []
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        futures = [executor.submit(fetch_jobs_for_run, run) for run in runs]
+        completed = 0
+        for future in as_completed(futures):
+            completed += 1
+            if completed % 50 == 0:
+                print(f"Fetched jobs for {completed}/{total_runs} runs...")
+            run_id, jobs = future.result()
+            if jobs:
+                all_jobs.extend(jobs)
+
+    print(f"Processing {len(all_jobs)} jobs...")
+
+    for job in all_jobs:
+        runner_name = job.get("runner_name")
+        if not runner_name:
+            continue
+
+        created_at = parse_time(job.get("created_at"))
+        started_at = parse_time(job.get("started_at"))
+        completed_at = parse_time(job.get("completed_at"))
+
+        if not started_at or not completed_at:
+            continue
+
+        duration = (completed_at - started_at).total_seconds()
+        queue_time = (started_at - created_at).total_seconds() if created_at else 0
+        job_info = {
+            "start": started_at,
+            "end": completed_at,
+            "created_at": created_at,
+            "duration": duration,
+            "queue_time": queue_time,
+            "job_name": job["name"],
+            "runner_name": runner_name,
+        }
+
+        # Use job labels directly (available in job data)
+        job_labels = job.get("labels", [])
+        for label in job_labels:
+            # Skip generic labels
+            if label in DEFAULT_LABELS_TO_IGNORE | GITHUB_HOSTED_LABELS:
+                continue
+            job_label_runners[label].add(runner_name)
+            label_jobs[label].append(job_info)
+
+    # Merge API runners and job-observed runners
+    # Prefer API count (online runners) when available
+    all_labels = set(api_label_runners.keys()) | set(job_label_runners.keys())
+
+    # Filter labels if specified
+    if runner_filter:
+        all_labels = {lbl for lbl in all_labels if runner_filter in lbl}
+
+    print(f"Tracking {len(all_labels)} runner labels: {sorted(all_labels)}")
+
+    # Calculate metrics per label
+    window_seconds = hours * 3600
+    window_end = datetime.now(timezone.utc)
+    window_start = window_end - timedelta(hours=hours)
+
+    results = []
+
+    for label in sorted(all_labels):
+        # Use API runner count if available, otherwise use job-observed count
+        if label in api_label_runners and api_label_runners[label]:
+            num_runners = len(api_label_runners[label])
+        elif label in job_label_runners:
+            num_runners = len(job_label_runners[label])
+        else:
+            num_runners = KNOWN_RUNNER_COUNTS.get(label, 1)
+
+        total_capacity_seconds = window_seconds * num_runners
+
+        jobs = label_jobs.get(label, [])
+        total_active_seconds = sum(j["duration"] for j in jobs)
+
+        utilization = (
+            (total_active_seconds / total_capacity_seconds * 100)
+            if total_capacity_seconds > 0
+            else 0
+        )
+        idle_seconds = total_capacity_seconds - total_active_seconds
+
+        # Calculate queue time metrics
+        queue_times = [j["queue_time"] for j in jobs if j["queue_time"] > 0]
+        avg_queue_time = sum(queue_times) / len(queue_times) if queue_times else 0
+        max_queue_time = max(queue_times) if queue_times else 0
+
+        # Calculate concurrency metrics
+        # First pass: get peak concurrent to determine effective capacity
+        concurrency_initial = calculate_concurrency_metrics(
+            jobs, window_start, window_end, num_runners
+        )
+
+        # Use observed peak as effective capacity if lower than API count
+        # This handles cases where not all runners are active all the time
+        effective_runners = min(num_runners, concurrency_initial["peak_concurrent"])
+        if effective_runners < num_runners and effective_runners > 0:
+            # Recalculate with effective capacity for accurate saturation
+            concurrency = calculate_concurrency_metrics(
+                jobs, window_start, window_end, effective_runners
+            )
+        else:
+            concurrency = concurrency_initial
+            effective_runners = num_runners
+
+        results.append(
+            {
+                "label": label,
+                "num_runners": num_runners,
+                "effective_runners": effective_runners,
+                "num_jobs": len(jobs),
+                "total_active_hours": total_active_seconds / 3600,
+                "total_idle_hours": idle_seconds / 3600,
+                "total_capacity_hours": total_capacity_seconds / 3600,
+                "utilization_pct": utilization,
+                "avg_queue_min": avg_queue_time / 60,
+                "max_queue_min": max_queue_time / 60,
+                # Concurrency metrics
+                "peak_concurrent": concurrency_initial["peak_concurrent"],
+                "avg_concurrent": concurrency["avg_concurrent"],
+                "saturation_hours": concurrency["saturation_seconds"] / 3600,
+                "saturation_pct": concurrency["saturation_pct"],
+                "peak_queue": concurrency["peak_queue"],
+            }
+        )
+
+    return results
+
+
+def format_report(results: list[dict], hours: int) -> str:
+    """Format results as markdown report."""
+    lines = [
+        "# Runner Utilization Report",
+        "",
+        f"**Time window:** Last {hours} hours",
+        f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
+        "",
+        "## Concurrency Analysis",
+        "",
+        "| Label | Runners (API/Effective) | Peak Concurrent | Avg Concurrent | Saturation Time | Peak Queue |",
+        "|-------|-------------------------|-----------------|----------------|-----------------|------------|",
+    ]
+
+    for r in results:
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+        runner_str = (
+            f"{r['num_runners']}/{effective}"
+            if effective != r["num_runners"]
+            else str(r["num_runners"])
+        )
+        lines.append(
+            f"| {r['label']} | {runner_str} | "
+            f"{r['peak_concurrent']} | "
+            f"{r['avg_concurrent']:.1f} ({avg_pct:.0f}%) | "
+            f"{r['saturation_hours']:.1f}h ({r['saturation_pct']:.0f}%) | "
+            f"{r['peak_queue']} jobs |"
+        )
+
+    # Add recommendations section
+    lines.extend(["", "## Recommendations", ""])
+    has_recommendations = False
+    for r in results:
+        label = r["label"]
+        saturation_pct = r["saturation_pct"]
+        peak_queue = r["peak_queue"]
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+
+        if saturation_pct > 50 or peak_queue > 5:
+            lines.append(
+                f"⚠️ **{label}**: High saturation ({saturation_pct:.0f}%) "
+                f"with queue buildup ({peak_queue} jobs). Consider adding runners."
+            )
+            has_recommendations = True
+        elif saturation_pct > 20 or peak_queue > 0:
+            lines.append(
+                f"📊 **{label}**: Moderate saturation ({saturation_pct:.0f}%), "
+                f"peak queue {peak_queue} jobs. Monitor for trends."
+            )
+            has_recommendations = True
+        elif avg_pct < 30 and r["num_jobs"] > 0:
+            lines.append(
+                f"💡 **{label}**: Low average utilization ({avg_pct:.0f}%). "
+                f"Runner pool may be oversized."
+            )
+            has_recommendations = True
+        else:
+            lines.append(f"✓ **{label}**: Healthy utilization with minimal queueing.")
+
+    if not has_recommendations and results:
+        lines.append("All runner pools have healthy utilization.")
+
+    # Add summary table
+    lines.extend(
+        [
+            "",
+            "## Summary by Runner Label",
+            "",
+            "| Label | Runners | Jobs | Active (hrs) | Utilization | Avg Queue | Max Queue |",
+            "|-------|---------|------|--------------|-------------|-----------|-----------|",
+        ]
+    )
+
+    for r in results:
+        utilization_bar = "█" * int(r["utilization_pct"] / 10) + "░" * (
+            10 - int(r["utilization_pct"] / 10)
+        )
+        lines.append(
+            f"| {r['label']} | {r['num_runners']} | {r['num_jobs']} | "
+            f"{r['total_active_hours']:.1f} | "
+            f"{r['utilization_pct']:.1f}% {utilization_bar} | "
+            f"{r['avg_queue_min']:.1f}m | {r['max_queue_min']:.1f}m |"
+        )
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate runner utilization report")
+    parser.add_argument("--repo", default="sgl-project/sglang", help="GitHub repo")
+    parser.add_argument("--hours", type=int, default=24, help="Time window in hours")
+    parser.add_argument(
+        "--filter", type=str, help="Filter runner labels (e.g., '5090', 'h200')"
+    )
+    parser.add_argument("--output", type=str, help="Output file (default: stdout)")
+    args = parser.parse_args()
+
+    results = calculate_utilization(args.repo, args.hours, args.filter)
+    report = format_report(results, args.hours)
+
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(report)
+        print(f"Report written to {args.output}")
+    else:
+        print(report)
+
+    # Also write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file:
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/save_metrics.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""Collect and save performance metrics from nightly benchmark results.
+
+This script reads benchmark result JSON files from performance profile directories
+and saves them with metadata for artifact collection in CI.
+
+Usage:
+    python3 scripts/ci/utils/save_metrics.py \
+        --gpu-config 8-gpu-h200 \
+        --partition 0 \
+        --run-id 12345678 \
+        --output test/metrics-8gpu-h200-partition-0.json
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_result_files(search_dirs: list[str]) -> list[str]:
+    """Find all results_*.json files in the given directories."""
+    result_files = set()
+    for search_dir in search_dirs:
+        if os.path.exists(search_dir):
+            pattern = os.path.join(search_dir, "**/results_*.json")
+            result_files.update(glob.glob(pattern, recursive=True))
+    return list(result_files)
+
+
+def parse_result_file(filepath: str) -> list[dict]:
+    """Parse a benchmark result JSON file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, list):
+            return data
+        return [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {filepath}: {e}")
+        return []
+
+
+def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
+    """Transform a benchmark result to the metrics schema.
+
+    Note: input_len and output_len are preserved here for the flat benchmarks list,
+    but are also used as grouping keys in benchmarks_by_io_len.
+    """
+    # Handle None values safely for numeric conversions
+    latency = result.get("latency")
+    last_ttft = result.get("last_ttft")
+
+    return {
+        "batch_size": result.get("batch_size"),
+        "input_len": result.get("input_len"),
+        "output_len": result.get("output_len"),
+        "latency_ms": latency * 1000 if latency is not None else None,
+        "input_throughput": result.get("input_throughput"),
+        "output_throughput": result.get("output_throughput"),
+        "overall_throughput": result.get("overall_throughput"),
+        "ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
+        "acc_length": result.get("acc_length"),
+    }
+
+
+def get_io_len_key(input_len: int, output_len: int) -> str:
+    """Generate a key for input/output length combination."""
+    return f"{input_len}_{output_len}"
+
+
+def group_results_by_model(
+    results: list[dict], gpu_config: str, partition: int
+) -> list[dict]:
+    """Group benchmark results by model, variant, and server_args.
+
+    Results are organized with two benchmark structures:
+    - benchmarks: flat list of all benchmarks (for backward compatibility)
+    - benchmarks_by_io_len: nested structure grouped by input/output length combinations
+    """
+    groups = {}
+
+    for result in results:
+        model_path = result.get("model_path", "unknown")
+        run_name = result.get("run_name", "default")
+        variant = run_name if run_name != "default" else None
+        server_args = result.get("server_args")
+        # Convert server_args list to tuple for use as dict key (lists are not hashable)
+        server_args_key = tuple(server_args) if server_args else None
+
+        key = (model_path, variant, server_args_key)
+        if key not in groups:
+            groups[key] = {
+                "gpu_config": gpu_config,
+                "partition": partition,
+                "model": model_path,
+                "variant": variant,
+                "server_args": server_args,
+                "benchmarks": [],
+                "benchmarks_by_io_len": {},
+            }
+
+        transformed = transform_benchmark_result(result, gpu_config, partition)
+
+        # Add to flat benchmarks list (backward compatibility)
+        groups[key]["benchmarks"].append(transformed)
+
+        # Add to nested benchmarks_by_io_len structure
+        input_len = result.get("input_len")
+        output_len = result.get("output_len")
+        if input_len is not None and output_len is not None:
+            io_key = get_io_len_key(input_len, output_len)
+            if io_key not in groups[key]["benchmarks_by_io_len"]:
+                groups[key]["benchmarks_by_io_len"][io_key] = {
+                    "input_len": input_len,
+                    "output_len": output_len,
+                    "benchmarks": [],
+                }
+            # For the nested structure, exclude input_len and output_len from individual benchmarks
+            # since they're already in the parent
+            nested_benchmark = {
+                k: v
+                for k, v in transformed.items()
+                if k not in ("input_len", "output_len")
+            }
+            groups[key]["benchmarks_by_io_len"][io_key]["benchmarks"].append(
+                nested_benchmark
+            )
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    partition: int,
+    run_id: str,
+    output_file: str,
+    search_dirs: list[str],
+) -> bool:
+    """Collect metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Find all result files
+    result_files = find_result_files(search_dirs)
+    print(f"Found {len(result_files)} result file(s)")
+
+    grouped = []
+    if not result_files:
+        print("No benchmark result files found")
+    else:
+        # Parse all result files
+        all_results = []
+        for filepath in sorted(result_files):
+            print(f"  Reading: {filepath}")
+            results = parse_result_file(filepath)
+            all_results.extend(results)
+        print(f"Total benchmark results: {len(all_results)}")
+
+        # Group by model/variant
+        grouped = group_results_by_model(all_results, gpu_config, partition)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "partition": partition,
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not result_files:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect performance metrics from benchmark results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
+    )
+    parser.add_argument(
+        "--partition",
+        type=int,
+        required=True,
+        help="Partition number (0, 1, 2, etc.)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--search-dir",
+        action="append",
+        default=[],
+        dest="search_dirs",
+        help="Directory to search for result files (can be specified multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    # Default search directories if none specified
+    search_dirs = args.search_dirs or [
+        "test/performance_profiles_8_gpu",
+        "test/performance_profiles_text_models",
+        "test/performance_profiles_vlms",
+        "test",
+        ".",
+    ]
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        partition=args.partition,
+        run_id=args.run_id,
+        output_file=args.output,
+        search_dirs=search_dirs,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

third_party/sglang/scripts/ci/utils/slash_command_handler.py

@@ -0,0 +1,870 @@
+import glob
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime, timezone
+
+import requests
+from github import Auth, Github
+
+# Configuration
+PERMISSIONS_FILE_PATH = ".github/CI_PERMISSIONS.json"
+
+
+def find_workflow_run_url(
+    gh_repo,
+    workflow_id,
+    ref,
+    target_stage,
+    token,
+    dispatch_time,
+    pr_head_sha=None,
+    max_wait=30,
+    test_command=None,
+):
+    """
+    Poll for the workflow run URL after dispatch.
+
+    Uses the dynamic run-name feature to identify runs:
+    - Fork PRs: display_title = "[stage-name] sha"
+    - Non-fork PRs: display_title = "[stage-name]"
+
+    Args:
+        gh_repo: PyGithub repository object
+        workflow_id: ID of the workflow that was dispatched
+        ref: Branch/ref the workflow was dispatched on
+        target_stage: The stage name we're looking for
+        token: GitHub API token
+        dispatch_time: Unix timestamp when dispatch was triggered
+        pr_head_sha: PR head SHA (for fork PRs, used to match display_title)
+        max_wait: Maximum seconds to wait for the run to appear
+
+    Returns:
+        The workflow run URL if found, None otherwise.
+    """
+    # Build expected display_title based on workflow's run-name.
+    # rerun-test includes test_command: "[rerun-test] <test_command> [<sha>]"
+    # Other workflows: "[stage-name] [<sha>]"
+    suffix = f" {test_command}" if test_command else ""
+    if pr_head_sha:
+        expected_title = f"[{target_stage}]{suffix} {pr_head_sha}"
+    else:
+        expected_title = f"[{target_stage}]{suffix}"
+
+    print(f"Looking for workflow run with display_title: {expected_title}")
+
+    for attempt in range(max_wait // 5):
+        time.sleep(5)
+
+        # Get recent workflow_dispatch runs for this workflow
+        runs_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{workflow_id}/runs"
+        runs_resp = requests.get(
+            runs_url,
+            params={"event": "workflow_dispatch", "branch": ref, "per_page": 10},
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+            },
+        )
+
+        if runs_resp.status_code != 200:
+            print(f"Failed to fetch workflow runs: {runs_resp.status_code}")
+            continue
+
+        for run in runs_resp.json().get("workflow_runs", []):
+            # Skip runs created before our dispatch (with 10s tolerance)
+            run_created = datetime.fromisoformat(
+                run["created_at"].replace("Z", "+00:00")
+            ).timestamp()
+            if run_created < dispatch_time - 10:
+                continue
+
+            # Match by display_title (set by workflow's run-name directive)
+            # This is immediately available, unlike job names which require waiting
+            display_title = run.get("display_title", "")
+            if display_title == expected_title:
+                print(
+                    f"Found matching workflow run: {run['id']} with title '{display_title}'"
+                )
+                return run["html_url"]
+
+    print(f"Could not find workflow run after {max_wait} seconds")
+    return None
+
+
+def get_env_var(name):
+    val = os.getenv(name)
+    if not val:
+        print(f"Error: Environment variable {name} not set.")
+        sys.exit(1)
+    return val
+
+
+def load_permissions(user_login):
+    """
+    Reads the permissions JSON from the local file system and returns
+    the permissions dict for the specific user.
+    """
+    try:
+        print(f"Loading permissions from {PERMISSIONS_FILE_PATH}...")
+        if not os.path.exists(PERMISSIONS_FILE_PATH):
+            print(f"Error: Permissions file not found at {PERMISSIONS_FILE_PATH}")
+            return None
+
+        with open(PERMISSIONS_FILE_PATH, "r") as f:
+            data = json.load(f)
+
+        user_perms = data.get(user_login)
+
+        if not user_perms:
+            print(f"User '{user_login}' not found in permissions file.")
+            return None
+
+        return user_perms
+
+    except Exception as e:
+        print(f"Failed to load or parse permissions file: {e}")
+        sys.exit(1)
+
+
+def has_sgl_kernel_changes(pr):
+    """
+    Check if the PR has changes to the sgl-kernel directory.
+    This is used to determine if we need a full workflow rerun
+    (to rebuild the kernel) vs just rerunning failed jobs.
+    """
+    try:
+        files = pr.get_files()
+        for f in files:
+            if f.filename.startswith("sgl-kernel/"):
+                return True
+        return False
+    except Exception as e:
+        print(f"Warning: Could not check PR files for sgl-kernel changes: {e}")
+        # Default to False to avoid unnecessary full reruns
+        return False
+
+
+def handle_tag_run_ci(gh_repo, pr, comment, user_perms, react_on_success=True):
+    """
+    Handles the /tag-run-ci-label command.
+    Returns True if action was taken, False otherwise.
+    """
+    if not user_perms.get("can_tag_run_ci_label", False):
+        print("Permission denied: can_tag_run_ci_label is false.")
+        return False
+
+    print("Permission granted. Adding 'run-ci' label.")
+    pr.add_to_labels("run-ci")
+
+    if react_on_success:
+        comment.create_reaction("+1")
+        print("Label added and comment reacted.")
+    else:
+        print("Label added (reaction suppressed).")
+
+    return True
+
+
+def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=True):
+    """
+    Handles the /rerun-failed-ci command.
+    Reruns workflows with 'failure' or 'skipped' conclusions.
+    Returns True if action was taken, False otherwise.
+    """
+    if not user_perms.get("can_rerun_failed_ci", False):
+        print("Permission denied: can_rerun_failed_ci is false.")
+        return False
+
+    print("Permission granted. Triggering rerun of failed or skipped workflows.")
+
+    # Check if PR has sgl-kernel changes - if so, we need full reruns
+    # to ensure sgl-kernel-build-wheels runs and produces fresh artifacts
+    sgl_kernel_changes = has_sgl_kernel_changes(pr)
+    if sgl_kernel_changes:
+        print("PR has sgl-kernel changes - will use full rerun to rebuild kernel")
+
+    # Get the SHA of the latest commit in the PR
+    head_sha = pr.head.sha
+    print(f"Checking workflows for commit: {head_sha}")
+
+    # List all workflow runs for this commit
+    runs = gh_repo.get_workflow_runs(head_sha=head_sha)
+
+    rerun_count = 0
+    for run in runs:
+        if run.status != "completed":
+            continue
+
+        if run.conclusion == "failure":
+            print(f"Rerunning failed workflow: {run.name} (ID: {run.id})")
+            try:
+                if sgl_kernel_changes:
+                    # Full rerun to ensure sgl-kernel-build-wheels runs
+                    # and produces fresh artifacts for dependent jobs
+                    run.rerun()
+                else:
+                    # Use rerun_failed_jobs for efficiency on failures
+                    run.rerun_failed_jobs()
+                rerun_count += 1
+            except Exception as e:
+                print(f"Failed to rerun workflow {run.id}: {e}")
+
+        elif run.conclusion == "skipped":
+            print(f"Rerunning skipped workflow: {run.name} (ID: {run.id})")
+            try:
+                # Skipped workflows don't have 'failed jobs', so we use full rerun()
+                run.rerun()
+                rerun_count += 1
+            except Exception as e:
+                print(f"Failed to rerun workflow {run.id}: {e}")
+
+    if rerun_count > 0:
+        print(f"Triggered rerun for {rerun_count} workflows.")
+        if react_on_success:
+            comment.create_reaction("+1")
+        return True
+    else:
+        print("No failed or skipped workflows found to rerun.")
+        return False
+
+
+def handle_rerun_stage(
+    gh_repo, pr, comment, user_perms, stage_name, token, react_on_success=True
+):
+    """
+    Handles the /rerun-stage <stage-name> command.
+    Triggers a workflow_dispatch to run only the specified stage, skipping dependencies.
+    Returns True if action was taken, False otherwise.
+    """
+    if not user_perms.get("can_rerun_stage", False):
+        print("Permission denied: can_rerun_stage is false.")
+        return False
+
+    if not stage_name:
+        print("Error: No stage name provided")
+        comment.create_reaction("confused")
+        pr.create_issue_comment(
+            f"❌ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
+            f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`"
+        )
+        return False
+
+    print(f"Permission granted. Triggering workflow_dispatch for stage '{stage_name}'.")
+
+    # Valid NVIDIA stage names that support target_stage
+    nvidia_stages = [
+        "stage-a-test-1-gpu-small",
+        "stage-a-test-cpu",
+        "stage-b-test-1-gpu-small",
+        "stage-b-test-1-gpu-large",
+        "stage-b-test-2-gpu-large",
+        "stage-b-test-4-gpu-b200",
+        "stage-c-test-4-gpu-h100",
+        "stage-c-test-8-gpu-h200",
+        "stage-c-test-8-gpu-h20",
+        "stage-c-test-4-gpu-b200",
+        "stage-c-test-4-gpu-gb200",
+        "stage-c-test-deepep-4-gpu-h100",
+        "stage-c-test-deepep-8-gpu-h200",
+        "multimodal-gen-test-1-gpu",
+        "multimodal-gen-test-2-gpu",
+        "multimodal-gen-test-1-b200",
+    ]
+
+    # Valid AMD stage names that support target_stage
+    amd_stages = [
+        "sgl-kernel-unit-test-amd",
+        "sgl-kernel-unit-test-2-gpu-amd",
+        "stage-a-test-1-gpu-small-amd",
+        "stage-b-test-1-gpu-small-amd",
+        "stage-b-test-1-gpu-small-amd-nondeterministic",
+        "stage-b-test-1-gpu-small-amd-mi35x",
+        "stage-b-test-1-gpu-large-amd",
+        "stage-b-test-2-gpu-large-amd",
+        "multimodal-gen-test-1-gpu-amd",
+        "multimodal-gen-test-2-gpu-amd",
+        "stage-c-test-large-8-gpu-amd",
+        "stage-c-test-large-8-gpu-amd-mi35x",
+    ]
+
+    valid_stages = nvidia_stages + amd_stages
+    is_amd_stage = stage_name in amd_stages
+
+    if stage_name not in valid_stages:
+        comment.create_reaction("confused")
+        pr.create_issue_comment(
+            f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
+            f"**NVIDIA stages:**\n"
+            + "\n".join(f"- `{s}`" for s in nvidia_stages)
+            + "\n\n**AMD stages:**\n"
+            + "\n".join(f"- `{s}`" for s in amd_stages)
+            + "\n\nOther stages will be added soon. For now, use `/rerun-failed-ci` for those stages."
+        )
+        return False
+
+    try:
+        # Get the appropriate workflow based on stage type
+        workflow_name = "PR Test (AMD)" if is_amd_stage else "PR Test"
+        workflows = gh_repo.get_workflows()
+        target_workflow = None
+        for wf in workflows:
+            if wf.name == workflow_name:
+                target_workflow = wf
+                break
+
+        if not target_workflow:
+            print(f"Error: {workflow_name} workflow not found")
+            return False
+
+        # Check if PR is from a fork by comparing repo owners
+        # Handle case where fork repo may have been deleted (pr.head.repo is None)
+        is_fork = (
+            pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
+        )
+        print(f"PR is from fork: {is_fork}")
+
+        # pr_head_sha is used for fork PRs (passed to workflow and used for URL lookup)
+        pr_head_sha = None
+
+        if is_fork:
+            # For fork PRs: dispatch on main and pass SHA as input
+            # This is needed because fork branch names don't exist in the main repo
+            ref = "main"
+            pr_head_sha = pr.head.sha
+            print(
+                f"Triggering {workflow_name} workflow on ref: {ref}, PR head SHA: {pr_head_sha}"
+            )
+            if is_amd_stage:
+                inputs = {
+                    "target_stage": stage_name,
+                    "pr_head_sha": pr_head_sha,
+                }
+            else:
+                inputs = {
+                    "target_stage": stage_name,
+                    "pr_head_sha": pr_head_sha,
+                }
+        else:
+            # For non-fork PRs: dispatch on the PR branch directly
+            # This allows testing workflow changes before merge
+            ref = pr.head.ref
+            print(f"Triggering {workflow_name} workflow on branch: {ref}")
+            if is_amd_stage:
+                inputs = {"target_stage": stage_name}
+            else:
+                inputs = {"target_stage": stage_name}
+
+        # Record dispatch time before triggering
+        dispatch_time = time.time()
+
+        # Use requests directly as PyGithub's create_dispatch only accepts HTTP 204
+        dispatch_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{target_workflow.id}/dispatches"
+        dispatch_resp = requests.post(
+            dispatch_url,
+            json={"ref": ref, "inputs": inputs},
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+            },
+        )
+        success = dispatch_resp.status_code in (200, 204)
+        if not success:
+            print(f"Dispatch failed: {dispatch_resp.status_code} {dispatch_resp.text}")
+
+        if success:
+            print(f"Successfully triggered workflow for stage '{stage_name}'")
+            if react_on_success:
+                comment.create_reaction("+1")
+
+                run_url = find_workflow_run_url(
+                    gh_repo,
+                    target_workflow.id,
+                    ref,
+                    stage_name,
+                    token,
+                    dispatch_time,
+                    pr_head_sha=pr_head_sha,
+                    max_wait=30,
+                )
+                if run_url:
+                    pr.create_issue_comment(
+                        f"✅ Triggered `{stage_name}` to run independently"
+                        f" (skipping dependencies)."
+                        f" [View workflow run]({run_url})"
+                    )
+                else:
+                    pr.create_issue_comment(
+                        f"✅ Triggered `{stage_name}` to run independently"
+                        f" (skipping dependencies).\n"
+                        f"⚠️ Could not retrieve workflow run URL. "
+                        f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
+                    )
+            return True
+        else:
+            print("Failed to trigger workflow_dispatch")
+            return False
+
+    except Exception as e:
+        print(f"Error triggering workflow_dispatch: {e}")
+        comment.create_reaction("confused")
+        pr.create_issue_comment(
+            f"❌ Failed to trigger workflow: {str(e)}\n\n"
+            f"Please check the logs or contact maintainers."
+        )
+        return False
+
+
+CUDA_SUITE_TO_RUNNER = {
+    "stage-a-test-1-gpu-small": "1-gpu-5090",
+    "stage-a-test-cpu": "ubuntu-latest",
+    "stage-b-test-1-gpu-small": "1-gpu-5090",
+    "stage-b-test-1-gpu-large": "1-gpu-h100",
+    "stage-b-test-2-gpu-large": "2-gpu-h100",
+    "stage-b-test-4-gpu-b200": "4-gpu-b200",
+    "stage-c-test-4-gpu-h100": "4-gpu-h100",
+    "stage-c-test-8-gpu-h200": "8-gpu-h200",
+    "stage-c-test-8-gpu-h20": "8-gpu-h20",
+    "stage-c-test-4-gpu-b200": "4-gpu-b200",
+    "stage-c-test-deepep-4-gpu-h100": "4-gpu-h100",
+    "stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
+}
+
+DEEPEP_SUITES = {
+    "stage-c-test-8-gpu-h20",
+    "stage-c-test-deepep-4-gpu-h100",
+    "stage-c-test-deepep-8-gpu-h200",
+}
+
+
+def resolve_test_file(file_part):
+    """
+    Resolve a user-provided file path to a path relative to test/.
+
+    Supports:
+    - Full path: test/registered/core/test_srt_endpoint.py
+    - Relative to test/: registered/core/test_srt_endpoint.py
+    - Bare filename: test_srt_endpoint.py (glob-matched, must be unique)
+
+    Returns (resolved_path, error_message). On success error_message is None.
+    """
+    if file_part.startswith("test/"):
+        file_part = file_part[len("test/") :]
+
+    if "/" not in file_part:
+        matches = glob.glob(f"test/registered/**/{file_part}", recursive=True)
+        if len(matches) == 0:
+            return (
+                None,
+                f"No test file found matching `{file_part}` under `test/registered/`.",
+            )
+        if len(matches) > 1:
+            match_list = "\n".join(f"- `{m}`" for m in sorted(matches))
+            return None, (
+                f"Ambiguous filename `{file_part}` — matched {len(matches)} files:\n\n"
+                f"{match_list}\n\n"
+                f"Please provide the full path, e.g. `/rerun-test {matches[0]}`"
+            )
+        return matches[0][len("test/") :], None
+
+    full_path = f"test/{file_part}"
+    if not os.path.isfile(full_path):
+        return None, f"File not found: `{full_path}`"
+    return file_part, None
+
+
+def detect_suite(file_path_from_test):
+    """
+    Read a test file and extract the suite from register_cuda_ci or register_cpu_ci.
+
+    Returns (suite_name, runner_label, use_deepep, is_cpu, error_message).
+    """
+    full_path = f"test/{file_path_from_test}"
+    with open(full_path, "r") as f:
+        content = f.read()
+
+    # Try CUDA first
+    match = re.search(
+        r'^[^#\n]*register_cuda_ci\([^)]*suite\s*=\s*["\']([^"\']+)["\']',
+        content,
+        re.MULTILINE,
+    )
+    if match:
+        suite = match.group(1)
+        runner = CUDA_SUITE_TO_RUNNER.get(suite)
+        if not runner:
+            known = ", ".join(f"`{s}`" for s in sorted(CUDA_SUITE_TO_RUNNER))
+            return (
+                suite,
+                None,
+                False,
+                False,
+                (
+                    f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n"
+                    f"Known suites: {known}"
+                ),
+            )
+        use_deepep = suite in DEEPEP_SUITES
+        return suite, runner, use_deepep, False, None
+
+    # Try CPU
+    match = re.search(
+        r'^[^#\n]*register_cpu_ci\([^)]*suite\s*=\s*["\']([^"\']+)["\']',
+        content,
+        re.MULTILINE,
+    )
+    if match:
+        suite = match.group(1)
+        return suite, "ubuntu-latest", False, True, None
+
+    return (
+        None,
+        None,
+        False,
+        False,
+        (
+            f"No `register_cuda_ci()` or `register_cpu_ci()` found in `{full_path}`.\n\n"
+            f"This file may not be a registered CI test."
+        ),
+    )
+
+
+def _resolve_test_spec(test_spec):
+    """
+    Resolve a single test spec into its components without dispatching.
+
+    Returns a dict with keys: spec, resolved_path, test_command, suite,
+    runner_label, use_deepep, is_cpu, error.
+    """
+    if "::" in test_spec:
+        file_part, test_selector = test_spec.split("::", 1)
+    else:
+        file_part = test_spec
+        test_selector = None
+
+    file_part = file_part.strip()
+    if test_selector:
+        test_selector = test_selector.strip()
+
+    resolved_path, err = resolve_test_file(file_part)
+    if err:
+        return {"spec": test_spec, "error": err}
+
+    suite, runner_label, use_deepep, is_cpu, err = detect_suite(resolved_path)
+    if err:
+        return {"spec": test_spec, "error": err}
+
+    test_command = resolved_path
+    if test_selector:
+        test_command = f"{resolved_path} {test_selector}"
+
+    print(
+        f"Resolved: file={resolved_path}, selector={test_selector}, "
+        f"suite={suite}, runner={runner_label}, deepep={use_deepep}, "
+        f"cpu={is_cpu}, command='{test_command}'"
+    )
+    return {
+        "spec": test_spec,
+        "test_command": test_command,
+        "suite": suite,
+        "runner_label": runner_label,
+        "use_deepep": use_deepep,
+        "is_cpu": is_cpu,
+        "error": None,
+    }
+
+
+def _dispatch_batch(gh_repo, pr, batch, token):
+    """
+    Dispatch a single workflow run for a batch of resolved test specs
+    that share the same (runner_label, use_deepep, is_cpu).
+
+    Returns a dict with keys: specs, success, test_commands, runner_label, run_url, error.
+    """
+    test_commands = [r["test_command"] for r in batch]
+    runner_label = batch[0]["runner_label"]
+    use_deepep = batch[0]["use_deepep"]
+    is_cpu = batch[0]["is_cpu"]
+
+    # Join multiple commands with newlines for the workflow to iterate over
+    combined_command = "\n".join(test_commands)
+
+    try:
+        workflow_name = "Rerun Test"
+        workflows = gh_repo.get_workflows()
+        target_workflow = None
+        for wf in workflows:
+            if wf.name == workflow_name:
+                target_workflow = wf
+                break
+
+        if not target_workflow:
+            return {
+                "specs": [r["spec"] for r in batch],
+                "success": False,
+                "error": f"{workflow_name} workflow not found",
+            }
+
+        is_fork = (
+            pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
+        )
+
+        pr_head_sha = None
+        inputs = {
+            "test_command": combined_command,
+            "runner_label": runner_label,
+            "use_deepep": str(use_deepep).lower(),
+            "is_cpu": str(is_cpu).lower(),
+        }
+        if is_fork:
+            ref = "main"
+            pr_head_sha = pr.head.sha
+            inputs["pr_head_sha"] = pr_head_sha
+        else:
+            ref = pr.head.ref
+
+        dispatch_time = time.time()
+
+        dispatch_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{target_workflow.id}/dispatches"
+        dispatch_resp = requests.post(
+            dispatch_url,
+            json={"ref": ref, "inputs": inputs},
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+            },
+        )
+        success = dispatch_resp.status_code in (200, 204)
+        if not success:
+            print(f"Dispatch failed: {dispatch_resp.status_code} {dispatch_resp.text}")
+            return {
+                "specs": [r["spec"] for r in batch],
+                "success": False,
+                "error": f"Dispatch failed: {dispatch_resp.status_code}",
+            }
+
+        print(f"Successfully triggered rerun-test: {combined_command}")
+
+        run_url = find_workflow_run_url(
+            gh_repo,
+            target_workflow.id,
+            ref,
+            "rerun-test",
+            token,
+            dispatch_time,
+            pr_head_sha=pr_head_sha,
+            max_wait=30,
+            test_command=combined_command,
+        )
+        return {
+            "specs": [r["spec"] for r in batch],
+            "success": True,
+            "test_commands": test_commands,
+            "runner_label": runner_label,
+            "run_url": run_url,
+        }
+
+    except Exception as e:
+        print(f"Error triggering rerun-test for batch: {e}")
+        return {
+            "specs": [r["spec"] for r in batch],
+            "success": False,
+            "error": str(e),
+        }
+
+
+def handle_rerun_test(gh_repo, pr, comment, user_perms, test_specs, token):
+    """
+    Handles the /rerun-test command. Resolves all test specs, groups them by
+    (runner_label, use_deepep, is_cpu), and dispatches one workflow per group.
+    """
+    # SECURITY: For fork PRs, only allow /rerun-test if the commenter has write+ permission.
+    # This command checks out and executes code from the PR branch on self-hosted GPU
+    # runners, so we must ensure the commenter is a trusted collaborator.
+    is_fork = pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
+    if is_fork:
+        commenter = comment.user.login
+        perm = gh_repo.get_collaborator_permission(commenter)
+        if perm not in ("admin", "write"):
+            print(f"Permission denied: /rerun-test on fork PR by {commenter}.")
+            comment.create_reaction("confused")
+            pr.create_issue_comment(
+                "❌ `/rerun-test` is not available for fork PRs unless the commenter "
+                "has write permission on the repo.\n\n"
+                "Please ask a maintainer to run this command, or use the normal CI flow."
+            )
+            return False
+        print(f"Fork PR, but commenter {commenter} has write+ permission. Proceeding.")
+
+    if not (
+        user_perms.get("can_rerun_test", False)
+        or user_perms.get("can_rerun_stage", False)
+    ):
+        print("Permission denied: neither can_rerun_test nor can_rerun_stage is true.")
+        return False
+
+    if not test_specs:
+        comment.create_reaction("confused")
+        pr.create_issue_comment(
+            "❌ Please specify a test: `/rerun-test <file>::<TestClass.test_method>`\n\n"
+            "Examples:\n"
+            "- `/rerun-test test/registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`\n"
+            "- `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint`\n"
+            "- `/rerun-test test_srt_endpoint.py`\n"
+            "- `/rerun-test test_a.py test_b.py test_c.py` (multiple tests)"
+        )
+        return False
+
+    # Phase 1: Resolve all specs
+    resolved = []
+    resolve_failures = []
+    for spec in test_specs:
+        r = _resolve_test_spec(spec)
+        if r.get("error"):
+            resolve_failures.append(r)
+        else:
+            resolved.append(r)
+
+    # Phase 2: Group by (runner_label, use_deepep, is_cpu)
+    groups = {}
+    for r in resolved:
+        key = (r["runner_label"], r["use_deepep"], r["is_cpu"])
+        groups.setdefault(key, []).append(r)
+
+    # Phase 3: Dispatch one workflow per group
+    dispatch_results = []
+    for batch in groups.values():
+        dispatch_results.append(_dispatch_batch(gh_repo, pr, batch, token))
+
+    # Build consolidated comment
+    lines = []
+    for dr in dispatch_results:
+        if dr["success"]:
+            cmds = "\n".join(
+                f"cd test/ && python3 {cmd}" for cmd in dr["test_commands"]
+            )
+            if dr.get("run_url"):
+                lines.append(
+                    f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): "
+                    f"[View workflow run]({dr['run_url']})\n"
+                    f"```\n{cmds}\n```"
+                )
+            else:
+                lines.append(
+                    f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}):\n"
+                    f"```\n{cmds}\n```\n"
+                    f"⚠️ Could not retrieve workflow run URL. "
+                    f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
+                )
+        else:
+            specs_str = ", ".join(f"`{s}`" for s in dr["specs"])
+            lines.append(f"❌ {specs_str}: {dr['error']}")
+
+    for r in resolve_failures:
+        lines.append(f"❌ `{r['spec']}`: {r['error']}")
+
+    body = "\n\n".join(lines)
+
+    successes = [dr for dr in dispatch_results if dr["success"]]
+    if successes:
+        comment.create_reaction("+1")
+    if not successes and (resolve_failures or dispatch_results):
+        comment.create_reaction("confused")
+
+    pr.create_issue_comment(body)
+    return len(successes) > 0
+
+
+def main():
+    # 1. Load Environment Variables
+    token = get_env_var("GITHUB_TOKEN")
+    repo_name = get_env_var("REPO_FULL_NAME")
+    pr_number = int(get_env_var("PR_NUMBER"))
+    comment_id = int(get_env_var("COMMENT_ID"))
+    comment_body = get_env_var("COMMENT_BODY").strip()
+    user_login = get_env_var("USER_LOGIN")
+
+    # 2. Load Permissions (local file check first to avoid unnecessary API calls)
+    user_perms = load_permissions(user_login)
+
+    # 3. Initialize GitHub API with Auth
+    auth = Auth.Token(token)
+    g = Github(auth=auth)
+
+    repo = g.get_repo(repo_name)
+    pr = repo.get_pull(pr_number)
+    comment = repo.get_issue(pr_number).get_comment(comment_id)
+
+    # PR authors can always rerun failed CI and rerun individual UTs on their own PRs,
+    # even if they are not listed in CI_PERMISSIONS.json.
+    # Note: /tag-run-ci-label and /rerun-stage still require CI_PERMISSIONS.json.
+    # Note: /rerun-test is blocked entirely for fork PRs in handle_rerun_test() itself.
+    if pr.user.login == user_login:
+        if user_perms is None:
+            print(
+                f"User {user_login} is the PR author (not in CI_PERMISSIONS.json). "
+                "Granting CI rerun permissions."
+            )
+            user_perms = {}
+        else:
+            print(
+                f"User {user_login} is the PR author and has existing CI permissions."
+            )
+        user_perms["can_rerun_failed_ci"] = True
+        user_perms["can_rerun_test"] = True
+
+    if not user_perms:
+        print(f"User {user_login} does not have any configured permissions. Exiting.")
+        return
+
+    # 4. Parse Command and Execute
+    first_line = comment_body.split("\n")[0].strip()
+
+    if first_line.startswith("/tag-run-ci-label"):
+        handle_tag_run_ci(repo, pr, comment, user_perms)
+
+    elif first_line.startswith("/rerun-failed-ci"):
+        handle_rerun_failed_ci(repo, pr, comment, user_perms)
+
+    elif first_line.startswith("/tag-and-rerun-ci"):
+        # Perform both actions, but suppress individual reactions
+        print("Processing combined command: /tag-and-rerun-ci")
+
+        tagged = handle_tag_run_ci(
+            repo, pr, comment, user_perms, react_on_success=False
+        )
+
+        # Wait for the label to propagate before triggering rerun
+        if tagged:
+            print("Waiting 5 seconds for label to propagate...")
+            time.sleep(5)
+
+        rerun = handle_rerun_failed_ci(
+            repo, pr, comment, user_perms, react_on_success=False
+        )
+
+        # If at least one action was successful, add the reaction here
+        if tagged or rerun:
+            comment.create_reaction("+1")
+            print("Combined command processed successfully; reaction added.")
+        else:
+            print("Combined command finished, but no actions were taken.")
+
+    elif first_line.startswith("/rerun-stage"):
+        # Extract stage name from command
+        parts = first_line.split(maxsplit=1)
+        stage_name = parts[1].strip() if len(parts) > 1 else None
+        handle_rerun_stage(repo, pr, comment, user_perms, stage_name, token)
+
+    elif first_line.startswith("/rerun-test"):
+        test_specs = first_line.split()[1:]
+        handle_rerun_test(repo, pr, comment, user_perms, test_specs or None, token)
+
+    else:
+        print(f"Unknown or ignored command: {first_line}")
+
+
+if __name__ == "__main__":
+    main()