fix(tokenizer): avoid GPU model imports for DeepSeek-V3.2

lightllm/models/deepseek3_2/model.py

@@ -1,4 +1,3 @@
-import copy
 from lightllm.models.deepseek2.model import Deepseek2TpPartModel
 from lightllm.models.deepseek3_2.layer_weights.transformer_layer_weight import Deepseek3_2TransformerLayerWeight
 from lightllm.models.deepseek3_2.layer_infer.transformer_layer_infer import Deepseek3_2TransformerLayerInfer
@@ -17,92 +16,3 @@ def _init_att_backend(self):
         self.prefill_att_backend = get_nsa_prefill_att_backend_class(index=0)(model=self)
         self.decode_att_backend = get_nsa_decode_att_backend_class(index=0)(model=self)
         return
-
-
-class DeepSeekV32Tokenizer:
-    """Tokenizer wrapper for DeepSeek-V3.2 that uses the Python-based
-    encoding_dsv32 module instead of Jinja chat templates.
-
-    DeepSeek-V3.2's tokenizer_config.json does not ship with a Jinja chat
-    template, so ``apply_chat_template`` would fail without either a manually
-    supplied ``--chat_template`` file or this wrapper.
-    """
-
-    def __init__(self, tokenizer):
-        self.tokenizer = tokenizer
-        # Cache added vocabulary for performance (HuggingFace can be slow).
-        self._added_vocab = None
-
-    # ------------------------------------------------------------------
-    # Attribute delegation – everything not overridden goes to the inner
-    # tokenizer so that encode/decode/vocab_size/eos_token_id/… all work.
-    # ------------------------------------------------------------------
-    def __getattr__(self, name):
-        return getattr(self.tokenizer, name)
-
-    def get_added_vocab(self):
-        if self._added_vocab is None:
-            self._added_vocab = self.tokenizer.get_added_vocab()
-        return self._added_vocab
-
-    # ------------------------------------------------------------------
-    # Core override: route apply_chat_template through encode_messages.
-    # ------------------------------------------------------------------
-    def apply_chat_template(
-        self,
-        conversation=None,
-        messages=None,
-        tools=None,
-        tokenize=False,
-        add_generation_prompt=True,
-        thinking=None,
-        **kwargs,
-    ):
-        from lightllm.models.deepseek3_2.encoding_dsv32 import encode_messages, render_tools
-
-        msgs = conversation if conversation is not None else messages
-        if msgs is None:
-            raise ValueError("Either 'conversation' or 'messages' must be provided")
-
-        # Deep copy to avoid mutating the caller's messages.
-        msgs = copy.deepcopy(msgs)
-
-        # Determine thinking mode.
-        thinking_mode = "thinking" if thinking else "chat"
-
-        # Inject tools into the first system message (or create one) so that
-        # encode_messages / render_message picks them up.
-        if tools:
-            # build_prompt passes tools as bare function dicts:
-            #   [{"name": "f", "description": "...", "parameters": {...}}]
-            # encoding_dsv32's render_message expects OpenAI wrapper format:
-            #   [{"type": "function", "function": {...}}]
-            wrapped_tools = []
-            for t in tools:
-                if "function" in t:
-                    wrapped_tools.append(t)
-                else:
-                    wrapped_tools.append({"type": "function", "function": t})
-
-            injected = False
-            for msg in msgs:
-                if msg.get("role") == "system":
-                    existing = msg.get("tools") or []
-                    msg["tools"] = existing + wrapped_tools
-                    injected = True
-                    break
-
-            if not injected:
-                # Prepend a system message that carries the tools.
-                msgs.insert(0, {"role": "system", "content": "", "tools": wrapped_tools})
-
-        prompt = encode_messages(
-            msgs,
-            thinking_mode=thinking_mode,
-            drop_thinking=kwargs.get("drop_thinking", True),
-            add_default_bos_token=kwargs.get("add_default_bos_token", True),
-        )
-
-        if tokenize:
-            return self.tokenizer.encode(prompt, add_special_tokens=False)
-        return prompt

lightllm/models/deepseek3_2/tokenizer.py

@@ -0,0 +1,90 @@
+import copy
+
+
+class DeepSeekV32Tokenizer:
+    """Tokenizer wrapper for DeepSeek-V3.2 that uses the Python-based
+    encoding_dsv32 module instead of Jinja chat templates.
+
+    DeepSeek-V3.2's tokenizer_config.json does not ship with a Jinja chat
+    template, so ``apply_chat_template`` would fail without either a manually
+    supplied ``--chat_template`` file or this wrapper.
+    """
+
+    def __init__(self, tokenizer):
+        self.tokenizer = tokenizer
+        # Cache added vocabulary for performance (HuggingFace can be slow).
+        self._added_vocab = None
+
+    # ------------------------------------------------------------------
+    # Attribute delegation – everything not overridden goes to the inner
+    # tokenizer so that encode/decode/vocab_size/eos_token_id/… all work.
+    # ------------------------------------------------------------------
+    def __getattr__(self, name):
+        return getattr(self.tokenizer, name)
+
+    def get_added_vocab(self):
+        if self._added_vocab is None:
+            self._added_vocab = self.tokenizer.get_added_vocab()
+        return self._added_vocab
+
+    # ------------------------------------------------------------------
+    # Core override: route apply_chat_template through encode_messages.
+    # ------------------------------------------------------------------
+    def apply_chat_template(
+        self,
+        conversation=None,
+        messages=None,
+        tools=None,
+        tokenize=False,
+        add_generation_prompt=True,
+        thinking=None,
+        **kwargs,
+    ):
+        from lightllm.models.deepseek3_2.encoding_dsv32 import encode_messages, render_tools
+
+        msgs = conversation if conversation is not None else messages
+        if msgs is None:
+            raise ValueError("Either 'conversation' or 'messages' must be provided")
+
+        # Deep copy to avoid mutating the caller's messages.
+        msgs = copy.deepcopy(msgs)
+
+        # Determine thinking mode.
+        thinking_mode = "thinking" if thinking else "chat"
+
+        # Inject tools into the first system message (or create one) so that
+        # encode_messages / render_message picks them up.
+        if tools:
+            # build_prompt passes tools as bare function dicts:
+            #   [{"name": "f", "description": "...", "parameters": {...}}]
+            # encoding_dsv32's render_message expects OpenAI wrapper format:
+            #   [{"type": "function", "function": {...}}]
+            wrapped_tools = []
+            for t in tools:
+                if "function" in t:
+                    wrapped_tools.append(t)
+                else:
+                    wrapped_tools.append({"type": "function", "function": t})
+
+            injected = False
+            for msg in msgs:
+                if msg.get("role") == "system":
+                    existing = msg.get("tools") or []
+                    msg["tools"] = existing + wrapped_tools
+                    injected = True
+                    break
+
+            if not injected:
+                # Prepend a system message that carries the tools.
+                msgs.insert(0, {"role": "system", "content": "", "tools": wrapped_tools})
+
+        prompt = encode_messages(
+            msgs,
+            thinking_mode=thinking_mode,
+            drop_thinking=kwargs.get("drop_thinking", True),
+            add_default_bos_token=kwargs.get("add_default_bos_token", True),
+        )
+
+        if tokenize:
+            return self.tokenizer.encode(prompt, add_special_tokens=False)
+        return prompt

lightllm/server/tokenizer.py

@@ -76,7 +76,7 @@ def get_tokenizer(
     # DeepSeek-V3.2 custom tokenizer mode: wraps the HF tokenizer with
     # a Python-based apply_chat_template that uses encoding_dsv32.py.
     if model_type == "deepseek_v32":
-        from ..models.deepseek3_2.model import DeepSeekV32Tokenizer
+        from ..models.deepseek3_2.tokenizer import DeepSeekV32Tokenizer
 
         hf_tokenizer = AutoTokenizer.from_pretrained(
             tokenizer_name, trust_remote_code=trust_remote_code, *args, **kwargs

unit_tests/server/test_tokenizer.py

@@ -25,7 +25,6 @@ def test_tokenizer_imports_are_isolated(model_type):
 from pathlib import Path
 import sys
 import tempfile
-from types import ModuleType
 from tokenizers import Tokenizer
 from tokenizers.models import WordLevel
 from transformers import AutoConfig, PreTrainedTokenizerFast
@@ -36,18 +35,18 @@ def find_spec(self, fullname, path=None, target=None):
         if fullname.startswith("lightllm.models."):
             allowed = {"lightllm.models.registry", "lightllm.models.draft_registry", "lightllm.models.builtin"}
             if model_type == "deepseek_v32":
-                allowed.add("lightllm.models.deepseek3_2")
+                allowed.update({
+                    "lightllm.models.deepseek3_2",
+                    "lightllm.models.deepseek3_2.tokenizer",
+                    "lightllm.models.deepseek3_2.encoding_dsv32",
+                })
             if fullname not in allowed:
                 raise AssertionError("Unexpected model import: " + fullname)
 sys.meta_path.insert(0, RejectUnrelatedModels())
 from lightllm.server import tokenizer as module
 assert not any(name.startswith("lightllm.models.") for name in sys.modules)
 
 if model_type == "deepseek_v32":
-    # Keep the real HF config registration, replacing only the implementation wrapper.
-    wrapper = ModuleType("lightllm.models.deepseek3_2.model")
-    wrapper.DeepSeekV32Tokenizer = lambda tokenizer: tokenizer
-    sys.modules[wrapper.__name__] = wrapper
     original = module.AutoTokenizer.from_pretrained
     def load(*args, **kwargs):
         assert AutoConfig.for_model("deepseek_v32").model_type == "deepseek_v32"
@@ -62,6 +61,20 @@ def load(*args, **kwargs):
     }))
     tokenizer = module.get_tokenizer(directory)
     assert tokenizer.encode("hello") == [0]
+    if model_type == "deepseek_v32":
+        assert tokenizer.__class__.__module__ == "lightllm.models.deepseek3_2.tokenizer"
+        prompt = tokenizer.apply_chat_template(messages=[{"role": "user", "content": "hello"}])
+        assert "hello" in prompt
+        thinking_prompt = tokenizer.apply_chat_template(
+            messages=[{"role": "user", "content": "hello"}], thinking=True
+        )
+        assert "<think>" in thinking_prompt
+        tool_prompt = tokenizer.apply_chat_template(
+            messages=[{"role": "user", "content": "hello"}],
+            tools=[{"name": "echo", "description": "Echo text", "parameters": {"type": "object"}}],
+        )
+        assert "<functions>" in tool_prompt and "echo" in tool_prompt
+        assert "lightllm.models.deepseek3_2.model" not in sys.modules
 """,
             model_type,
         ],