From d5ed2e69d85be177d9428146bf08f13d4e16adde Mon Sep 17 00:00:00 2001 From: lyt Date: Tue, 28 Apr 2026 17:37:57 +0000 Subject: [PATCH 1/2] Making sgl-kernel work by chaging torch back to 2.9.1 --- python/pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index b05f66cfc32e..465bd08c5f67 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "blobfile==3.0.0", "build", "compressed-tensors", - "cuda-python>=13.1,<13.3", + "cuda-python>=13.0", "decord2", "datasets", "einops", @@ -64,11 +64,11 @@ dependencies = [ "tiktoken", "timm==1.0.16", "torch_memory_saver==0.0.9", - "torch>=2.10,<2.12", + "torch==2.9.1", "torchao==0.9.0", - "torchaudio>=2.10,<2.12", + "torchaudio==2.9.1", "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. - "torchvision>=0.25,<0.27", + "torchvision", "tqdm", "transformers==4.57.1", "uvicorn", From d02dc180e7337c97650ccf2ccb0b803c245ecf7c Mon Sep 17 00:00:00 2001 From: lyt Date: Tue, 23 Jun 2026 06:31:53 +0000 Subject: [PATCH 2/2] Support PagedMoE storage root for GLM MoE --- python/sglang/srt/configs/__init__.py | 3 ++- python/sglang/srt/configs/deepseekvl2.py | 4 +++ python/sglang/srt/layers/moe/kt_ep_wrapper.py | 11 ++++++-- python/sglang/srt/server_args.py | 6 +++++ .../sglang/srt/utils/hf_transformers_utils.py | 27 ++++++++++++++++++- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index e14cd2929469..08e6a8bee381 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -2,7 +2,7 @@ from sglang.srt.configs.bailing_hybrid import BailingHybridConfig from sglang.srt.configs.chatglm import ChatGLMConfig from sglang.srt.configs.dbrx import DbrxConfig -from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config +from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config, GlmMoeDsaConfig from sglang.srt.configs.dots_ocr import DotsOCRConfig from sglang.srt.configs.dots_vlm import DotsVLMConfig from sglang.srt.configs.exaone import ExaoneConfig @@ -38,6 +38,7 @@ "ChatGLMConfig", "DbrxConfig", "DeepseekVL2Config", + "GlmMoeDsaConfig", "LongcatFlashConfig", "MultiModalityConfig", "KimiVLConfig", diff --git a/python/sglang/srt/configs/deepseekvl2.py b/python/sglang/srt/configs/deepseekvl2.py index 9621f058bf63..8ef529ee3207 100644 --- a/python/sglang/srt/configs/deepseekvl2.py +++ b/python/sglang/srt/configs/deepseekvl2.py @@ -647,6 +647,10 @@ def __init__( ) +class GlmMoeDsaConfig(DeepseekV2Config): + model_type = "glm_moe_dsa" + + class DeepseekVL2Config(PretrainedConfig): model_type = "deepseek_vl_v2" vision_config: DeepseekVL2VisionEncoderConfig diff --git a/python/sglang/srt/layers/moe/kt_ep_wrapper.py b/python/sglang/srt/layers/moe/kt_ep_wrapper.py index 799724cbcd39..e89bdc63be45 100644 --- a/python/sglang/srt/layers/moe/kt_ep_wrapper.py +++ b/python/sglang/srt/layers/moe/kt_ep_wrapper.py @@ -1646,7 +1646,14 @@ def create_kt_config_from_server_args( if is_kt_ep_wrapper_disabled(): return None - if server_args.kt_weight_path is None: + kt_method = (server_args.kt_method or "").upper() + weight_path = ( + server_args.pagedmoe_storage_root + if kt_method == "PAGEDMOE" and server_args.pagedmoe_storage_root is not None + else server_args.kt_weight_path + ) + + if weight_path is None: return None # Get GPU experts masks (initializes if needed) @@ -1669,7 +1676,7 @@ def create_kt_config_from_server_args( cpuinfer_threads=server_args.kt_cpuinfer, threadpool_count=server_args.kt_threadpool_count, numa_nodes=server_args.kt_numa_nodes, - weight_path=server_args.kt_weight_path, + weight_path=weight_path, chunked_prefill_size=server_args.chunked_prefill_size, method=server_args.kt_method, max_deferred_experts_per_token=server_args.kt_max_deferred_experts_per_token, diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 80d43d0ad093..e59d8d33e6ea 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -551,6 +551,7 @@ class ServerArgs: # Ktransformers/AMX expert parallelism kt_weight_path: Optional[str] = None + pagedmoe_storage_root: Optional[str] = None kt_method: Optional[str] = None kt_cpuinfer: Optional[int] = None kt_threadpool_count: Optional[int] = None @@ -4436,6 +4437,11 @@ def add_cli_args(parser: argparse.ArgumentParser): type=str, help="[ktransformers parameter] The path of the quantized expert weights for amx kernel. A local folder.", ) + parser.add_argument( + "--pagedmoe-storage-root", + type=str, + help="[pagedmoe parameter] The root directory containing pagedmoe manifest.json and layer storage.", + ) parser.add_argument( "--kt-method", type=str, diff --git a/python/sglang/srt/utils/hf_transformers_utils.py b/python/sglang/srt/utils/hf_transformers_utils.py index 5ff9bbc732e4..4f7f5ea601a6 100644 --- a/python/sglang/srt/utils/hf_transformers_utils.py +++ b/python/sglang/srt/utils/hf_transformers_utils.py @@ -53,6 +53,7 @@ DotsVLMConfig, ExaoneConfig, FalconH1Config, + GlmMoeDsaConfig, GraniteMoeHybridConfig, JetNemotronConfig, JetVLMConfig, @@ -86,6 +87,7 @@ ChatGLMConfig, DbrxConfig, ExaoneConfig, + GlmMoeDsaConfig, DeepseekVL2Config, MultiModalityConfig, KimiVLConfig, @@ -541,9 +543,32 @@ def get_tokenizer( ) raise RuntimeError(err_msg) from e except ValueError as e: + tokenizer_json = Path(tokenizer_name) / "tokenizer.json" + tokenizer_config = Path(tokenizer_name) / "tokenizer_config.json" + if "TokenizersBackend" in str(e) and tokenizer_json.exists(): + tokenizer_kwargs: Dict[str, Any] = { + "tokenizer_file": str(tokenizer_json), + "clean_up_tokenization_spaces": False, + } + if tokenizer_config.exists(): + with tokenizer_config.open() as f: + tokenizer_config_data = json.load(f) + for key in ( + "bos_token", + "eos_token", + "pad_token", + "unk_token", + "model_max_length", + ): + if key in tokenizer_config_data: + tokenizer_kwargs[key] = tokenizer_config_data[key] + tokenizer = PreTrainedTokenizerFast(**tokenizer_kwargs) + logging.getLogger(tokenizer.__class__.__module__).addFilter( + TokenizerWarningsFilter() + ) # If the error pertains to the tokenizer class not existing or not # currently being imported, suggest using the --trust-remote-code flag. - if not trust_remote_code and ( + elif not trust_remote_code and ( "does not exist or is not currently imported." in str(e) or "requires you to execute the tokenizer file" in str(e) ):