diff --git a/lightllm/models/deepseek3_2/model.py b/lightllm/models/deepseek3_2/model.py index 415e8826f8..57b6739e6f 100644 --- a/lightllm/models/deepseek3_2/model.py +++ b/lightllm/models/deepseek3_2/model.py @@ -1,4 +1,3 @@ -import copy from lightllm.models.deepseek2.model import Deepseek2TpPartModel from lightllm.models.deepseek3_2.layer_weights.transformer_layer_weight import Deepseek3_2TransformerLayerWeight from lightllm.models.deepseek3_2.layer_infer.transformer_layer_infer import Deepseek3_2TransformerLayerInfer @@ -17,92 +16,3 @@ def _init_att_backend(self): self.prefill_att_backend = get_nsa_prefill_att_backend_class(index=0)(model=self) self.decode_att_backend = get_nsa_decode_att_backend_class(index=0)(model=self) return - - -class DeepSeekV32Tokenizer: - """Tokenizer wrapper for DeepSeek-V3.2 that uses the Python-based - encoding_dsv32 module instead of Jinja chat templates. - - DeepSeek-V3.2's tokenizer_config.json does not ship with a Jinja chat - template, so ``apply_chat_template`` would fail without either a manually - supplied ``--chat_template`` file or this wrapper. - """ - - def __init__(self, tokenizer): - self.tokenizer = tokenizer - # Cache added vocabulary for performance (HuggingFace can be slow). - self._added_vocab = None - - # ------------------------------------------------------------------ - # Attribute delegation – everything not overridden goes to the inner - # tokenizer so that encode/decode/vocab_size/eos_token_id/… all work. - # ------------------------------------------------------------------ - def __getattr__(self, name): - return getattr(self.tokenizer, name) - - def get_added_vocab(self): - if self._added_vocab is None: - self._added_vocab = self.tokenizer.get_added_vocab() - return self._added_vocab - - # ------------------------------------------------------------------ - # Core override: route apply_chat_template through encode_messages. - # ------------------------------------------------------------------ - def apply_chat_template( - self, - conversation=None, - messages=None, - tools=None, - tokenize=False, - add_generation_prompt=True, - thinking=None, - **kwargs, - ): - from lightllm.models.deepseek3_2.encoding_dsv32 import encode_messages, render_tools - - msgs = conversation if conversation is not None else messages - if msgs is None: - raise ValueError("Either 'conversation' or 'messages' must be provided") - - # Deep copy to avoid mutating the caller's messages. - msgs = copy.deepcopy(msgs) - - # Determine thinking mode. - thinking_mode = "thinking" if thinking else "chat" - - # Inject tools into the first system message (or create one) so that - # encode_messages / render_message picks them up. - if tools: - # build_prompt passes tools as bare function dicts: - # [{"name": "f", "description": "...", "parameters": {...}}] - # encoding_dsv32's render_message expects OpenAI wrapper format: - # [{"type": "function", "function": {...}}] - wrapped_tools = [] - for t in tools: - if "function" in t: - wrapped_tools.append(t) - else: - wrapped_tools.append({"type": "function", "function": t}) - - injected = False - for msg in msgs: - if msg.get("role") == "system": - existing = msg.get("tools") or [] - msg["tools"] = existing + wrapped_tools - injected = True - break - - if not injected: - # Prepend a system message that carries the tools. - msgs.insert(0, {"role": "system", "content": "", "tools": wrapped_tools}) - - prompt = encode_messages( - msgs, - thinking_mode=thinking_mode, - drop_thinking=kwargs.get("drop_thinking", True), - add_default_bos_token=kwargs.get("add_default_bos_token", True), - ) - - if tokenize: - return self.tokenizer.encode(prompt, add_special_tokens=False) - return prompt diff --git a/lightllm/models/deepseek3_2/tokenizer.py b/lightllm/models/deepseek3_2/tokenizer.py new file mode 100644 index 0000000000..18ac4f1ab3 --- /dev/null +++ b/lightllm/models/deepseek3_2/tokenizer.py @@ -0,0 +1,90 @@ +import copy + + +class DeepSeekV32Tokenizer: + """Tokenizer wrapper for DeepSeek-V3.2 that uses the Python-based + encoding_dsv32 module instead of Jinja chat templates. + + DeepSeek-V3.2's tokenizer_config.json does not ship with a Jinja chat + template, so ``apply_chat_template`` would fail without either a manually + supplied ``--chat_template`` file or this wrapper. + """ + + def __init__(self, tokenizer): + self.tokenizer = tokenizer + # Cache added vocabulary for performance (HuggingFace can be slow). + self._added_vocab = None + + # ------------------------------------------------------------------ + # Attribute delegation – everything not overridden goes to the inner + # tokenizer so that encode/decode/vocab_size/eos_token_id/… all work. + # ------------------------------------------------------------------ + def __getattr__(self, name): + return getattr(self.tokenizer, name) + + def get_added_vocab(self): + if self._added_vocab is None: + self._added_vocab = self.tokenizer.get_added_vocab() + return self._added_vocab + + # ------------------------------------------------------------------ + # Core override: route apply_chat_template through encode_messages. + # ------------------------------------------------------------------ + def apply_chat_template( + self, + conversation=None, + messages=None, + tools=None, + tokenize=False, + add_generation_prompt=True, + thinking=None, + **kwargs, + ): + from lightllm.models.deepseek3_2.encoding_dsv32 import encode_messages, render_tools + + msgs = conversation if conversation is not None else messages + if msgs is None: + raise ValueError("Either 'conversation' or 'messages' must be provided") + + # Deep copy to avoid mutating the caller's messages. + msgs = copy.deepcopy(msgs) + + # Determine thinking mode. + thinking_mode = "thinking" if thinking else "chat" + + # Inject tools into the first system message (or create one) so that + # encode_messages / render_message picks them up. + if tools: + # build_prompt passes tools as bare function dicts: + # [{"name": "f", "description": "...", "parameters": {...}}] + # encoding_dsv32's render_message expects OpenAI wrapper format: + # [{"type": "function", "function": {...}}] + wrapped_tools = [] + for t in tools: + if "function" in t: + wrapped_tools.append(t) + else: + wrapped_tools.append({"type": "function", "function": t}) + + injected = False + for msg in msgs: + if msg.get("role") == "system": + existing = msg.get("tools") or [] + msg["tools"] = existing + wrapped_tools + injected = True + break + + if not injected: + # Prepend a system message that carries the tools. + msgs.insert(0, {"role": "system", "content": "", "tools": wrapped_tools}) + + prompt = encode_messages( + msgs, + thinking_mode=thinking_mode, + drop_thinking=kwargs.get("drop_thinking", True), + add_default_bos_token=kwargs.get("add_default_bos_token", True), + ) + + if tokenize: + return self.tokenizer.encode(prompt, add_special_tokens=False) + return prompt diff --git a/lightllm/server/tokenizer.py b/lightllm/server/tokenizer.py index 5dadaf0b13..2b5f1ffd54 100644 --- a/lightllm/server/tokenizer.py +++ b/lightllm/server/tokenizer.py @@ -76,7 +76,7 @@ def get_tokenizer( # DeepSeek-V3.2 custom tokenizer mode: wraps the HF tokenizer with # a Python-based apply_chat_template that uses encoding_dsv32.py. if model_type == "deepseek_v32": - from ..models.deepseek3_2.model import DeepSeekV32Tokenizer + from ..models.deepseek3_2.tokenizer import DeepSeekV32Tokenizer hf_tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, trust_remote_code=trust_remote_code, *args, **kwargs diff --git a/unit_tests/server/test_tokenizer.py b/unit_tests/server/test_tokenizer.py index 6f6d22da55..a6414291fe 100644 --- a/unit_tests/server/test_tokenizer.py +++ b/unit_tests/server/test_tokenizer.py @@ -25,7 +25,6 @@ def test_tokenizer_imports_are_isolated(model_type): from pathlib import Path import sys import tempfile -from types import ModuleType from tokenizers import Tokenizer from tokenizers.models import WordLevel from transformers import AutoConfig, PreTrainedTokenizerFast @@ -36,7 +35,11 @@ def find_spec(self, fullname, path=None, target=None): if fullname.startswith("lightllm.models."): allowed = {"lightllm.models.registry", "lightllm.models.draft_registry", "lightllm.models.builtin"} if model_type == "deepseek_v32": - allowed.add("lightllm.models.deepseek3_2") + allowed.update({ + "lightllm.models.deepseek3_2", + "lightllm.models.deepseek3_2.tokenizer", + "lightllm.models.deepseek3_2.encoding_dsv32", + }) if fullname not in allowed: raise AssertionError("Unexpected model import: " + fullname) sys.meta_path.insert(0, RejectUnrelatedModels()) @@ -44,10 +47,6 @@ def find_spec(self, fullname, path=None, target=None): assert not any(name.startswith("lightllm.models.") for name in sys.modules) if model_type == "deepseek_v32": - # Keep the real HF config registration, replacing only the implementation wrapper. - wrapper = ModuleType("lightllm.models.deepseek3_2.model") - wrapper.DeepSeekV32Tokenizer = lambda tokenizer: tokenizer - sys.modules[wrapper.__name__] = wrapper original = module.AutoTokenizer.from_pretrained def load(*args, **kwargs): assert AutoConfig.for_model("deepseek_v32").model_type == "deepseek_v32" @@ -62,6 +61,20 @@ def load(*args, **kwargs): })) tokenizer = module.get_tokenizer(directory) assert tokenizer.encode("hello") == [0] + if model_type == "deepseek_v32": + assert tokenizer.__class__.__module__ == "lightllm.models.deepseek3_2.tokenizer" + prompt = tokenizer.apply_chat_template(messages=[{"role": "user", "content": "hello"}]) + assert "hello" in prompt + thinking_prompt = tokenizer.apply_chat_template( + messages=[{"role": "user", "content": "hello"}], thinking=True + ) + assert "" in thinking_prompt + tool_prompt = tokenizer.apply_chat_template( + messages=[{"role": "user", "content": "hello"}], + tools=[{"name": "echo", "description": "Echo text", "parameters": {"type": "object"}}], + ) + assert "" in tool_prompt and "echo" in tool_prompt + assert "lightllm.models.deepseek3_2.model" not in sys.modules """, model_type, ],