chore: vendor sglang v0.5.10 snapshot
This commit is contained in:
211
third_party/sglang/test/manual/layers/moe/test_moe_runners_1gpu.py
vendored
Normal file
211
third_party/sglang/test/manual/layers/moe/test_moe_runners_1gpu.py
vendored
Normal file
@@ -0,0 +1,211 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE,
|
||||
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestMoERunner(CustomTestCase):
|
||||
BASE_URL = DEFAULT_URL_FOR_TEST
|
||||
TIMEOUT = 6000
|
||||
DEFAULT_EVAL_KWARGS = {
|
||||
"eval_name": "mmlu",
|
||||
"num_examples": 5,
|
||||
"num_threads": 1,
|
||||
}
|
||||
|
||||
CONFIGS = {
|
||||
"moe_runner_auto": {
|
||||
"model": DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"triton",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_triton": {
|
||||
"model": DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"triton",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_triton_kernel": {
|
||||
"model": DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"triton_kernel",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_flashinfer_cutlass": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4, # requires model with modelopt_fp4 quantization
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_cutlass",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_deep_gemm": {
|
||||
"model": DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"deep_gemm",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_flashinfer_trtllm": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE, # modelopt_fp4 or fp8 quantization is required for Flashinfer trtllm MOE
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
],
|
||||
},
|
||||
"moe_runner_flashinfer_mxfp4": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_mxfp4",
|
||||
"--quantization",
|
||||
"mxfp4",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_flashinfer_cutedsl": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_cutedsl",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_cutlass": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"cutlass",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
"moe_runner_cutlass_fp8": {
|
||||
"model": DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE,
|
||||
"timeout": 3600,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"cutlass",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
"--disable-cuda-graph",
|
||||
],
|
||||
},
|
||||
"moe_runner_speculative": {
|
||||
"model": DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"triton",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT,
|
||||
"--speculative-moe-runner-backend",
|
||||
"triton",
|
||||
"--speculative-num-steps",
|
||||
"2",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--attention-backend",
|
||||
"torch_native",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
def _run_config(self, config: dict) -> None:
|
||||
model = config["model"]
|
||||
other_args = config.get("other_args", [])
|
||||
eval_kwargs = self.DEFAULT_EVAL_KWARGS
|
||||
env = dict(os.environ)
|
||||
env["SGLANG_ENABLE_JIT_DEEPGEMM"] = "1"
|
||||
env["SGLANG_JIT_DEEPGEMM_PRECOMPILE"] = "0"
|
||||
env.update(config.get("env_overrides", {}))
|
||||
timeout = config.get("timeout", self.TIMEOUT)
|
||||
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
self.BASE_URL,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
try:
|
||||
args = SimpleNamespace(
|
||||
base_url=self.BASE_URL,
|
||||
model=model,
|
||||
**eval_kwargs,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreaterEqual(metrics["score"], 0.48)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
for _name, _cfg in TestMoERunner.CONFIGS.items():
|
||||
setattr(
|
||||
TestMoERunner,
|
||||
f"test_{_name}",
|
||||
(lambda self, cfg=_cfg: self._run_config(cfg)),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
116
third_party/sglang/test/manual/layers/moe/test_moe_runners_4gpu.py
vendored
Normal file
116
third_party/sglang/test/manual/layers/moe/test_moe_runners_4gpu.py
vendored
Normal file
@@ -0,0 +1,116 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestMoERunner4GPU(CustomTestCase):
|
||||
BASE_URL = DEFAULT_URL_FOR_TEST
|
||||
TIMEOUT = 6000
|
||||
DEFAULT_EVAL_KWARGS = {
|
||||
"eval_name": "mmlu",
|
||||
"num_examples": 5,
|
||||
"num_threads": 1,
|
||||
}
|
||||
|
||||
CONFIGS = {
|
||||
"moe_runner_cutlass_w4a8": {
|
||||
"model": "tencent/DeepSeek-V3.1-Terminus-W4AFP8", # FP8 W8A8 MoE model
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"cutlass",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
"--tp-size",
|
||||
"4",
|
||||
],
|
||||
},
|
||||
"moe_runner_cutlass_w4a8_deepep_normal": {
|
||||
"model": "tencent/DeepSeek-V3.1-Terminus-W4AFP8", # FP8 W8A8 MoE model
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"cutlass",
|
||||
"--moe-a2a-backend",
|
||||
"deepep",
|
||||
"--deepep-mode",
|
||||
"normal",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
"--tp-size",
|
||||
"4",
|
||||
],
|
||||
},
|
||||
"moe_runner_cutlass_w4a8_deepep_ll": {
|
||||
"model": "tencent/DeepSeek-V3.1-Terminus-W4AFP8", # FP8 W8A8 MoE model
|
||||
"env_overrides": {"SGLANG_DEEPEP_BF16_DISPATCH": "1"},
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--moe-runner-backend",
|
||||
"cutlass",
|
||||
"--moe-a2a-backend",
|
||||
"deepep",
|
||||
"--deepep-mode",
|
||||
"low_latency",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--sampling-backend",
|
||||
"pytorch",
|
||||
"--tp-size",
|
||||
"4",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
def _run_config(self, config: dict) -> None:
|
||||
model = config["model"]
|
||||
other_args = config.get("other_args", [])
|
||||
eval_kwargs = self.DEFAULT_EVAL_KWARGS
|
||||
env = dict(os.environ)
|
||||
env["SGLANG_ENABLE_JIT_DEEPGEMM"] = "1"
|
||||
env["SGLANG_JIT_DEEPGEMM_PRECOMPILE"] = "0"
|
||||
env.update(config.get("env_overrides", {}))
|
||||
timeout = config.get("timeout", self.TIMEOUT)
|
||||
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
self.BASE_URL,
|
||||
timeout=timeout,
|
||||
other_args=other_args,
|
||||
env=env,
|
||||
)
|
||||
try:
|
||||
args = SimpleNamespace(
|
||||
base_url=self.BASE_URL,
|
||||
model=model,
|
||||
**eval_kwargs,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreaterEqual(metrics["score"], 0.48)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
for _name, _cfg in TestMoERunner4GPU.CONFIGS.items():
|
||||
setattr(
|
||||
TestMoERunner4GPU,
|
||||
f"test_{_name}",
|
||||
(lambda self, cfg=_cfg: self._run_config(cfg)),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user