From 531f5ab4cfbb4588eac9549ae99126661d0aaf62 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:03:53 +0000 Subject: [PATCH 1/2] fix(reasoning): ignore preclosed prompt markers Do not seed streaming reasoning state when the latest prompt thinking marker is already followed by its matching closing marker. This keeps direct Gemma 4 output in content when its template disables thinking with a preclosed channel. Assisted-by: Codex:gpt-5 --- core/http/endpoints/openai/chat.go | 3 +++ .../endpoints/openai/chat_stream_workers.go | 6 +++++ core/http/endpoints/openai/realtime.go | 3 +++ core/http/endpoints/openai/realtime_stream.go | 3 +++ .../http/endpoints/openresponses/responses.go | 6 +++++ pkg/reasoning/reasoning.go | 23 ++++++++++++++++++- pkg/reasoning/reasoning_test.go | 23 +++++++++++++++++++ 7 files changed, 66 insertions(+), 1 deletion(-) diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f863631f6630..8309bee10e59 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -696,6 +696,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &config.ReasoningConfig) + if config.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig) + } xlog.Debug("Thinking start token", "thinkingStartToken", thinkingStartToken, "template", template) diff --git a/core/http/endpoints/openai/chat_stream_workers.go b/core/http/endpoints/openai/chat_stream_workers.go index 839f40676567..f013b39a7db9 100644 --- a/core/http/endpoints/openai/chat_stream_workers.go +++ b/core/http/endpoints/openai/chat_stream_workers.go @@ -150,6 +150,9 @@ func processStream( template = s } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) // preferAutoparser is sticky: once the C++ autoparser has ever classified @@ -248,6 +251,9 @@ func processStreamWithTools( template = prompt } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) result := "" diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index b22e1a8dde7e..b52f58eb984e 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2289,6 +2289,9 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa template = config.TemplateConfig.Chat } thinkingStartToken := reasoning.DetectThinkingStartToken(template, &config.ReasoningConfig) + if config.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig) + } // When the C++ autoparser emitted ChatDeltas with actionable data, // prefer them — the backend clears Reply.Message in that path and diff --git a/core/http/endpoints/openai/realtime_stream.go b/core/http/endpoints/openai/realtime_stream.go index 7c37f7affc0a..debe2de2b329 100644 --- a/core/http/endpoints/openai/realtime_stream.go +++ b/core/http/endpoints/openai/realtime_stream.go @@ -145,6 +145,9 @@ func streamLLMResponse(ctx context.Context, session *Session, conv *Conversation template = llmCfg.TemplateConfig.Chat } thinkingStartToken := reasoning.DetectThinkingStartToken(template, &llmCfg.ReasoningConfig) + if llmCfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &llmCfg.ReasoningConfig) + } // The autoparser (tokenizer-template path) already delivers reasoning-free // content. Prefilling the thinking start token here would re-tag that clean diff --git a/core/http/endpoints/openresponses/responses.go b/core/http/endpoints/openresponses/responses.go index d86780fbd2c4..d110ecf8f11a 100644 --- a/core/http/endpoints/openresponses/responses.go +++ b/core/http/endpoints/openresponses/responses.go @@ -1359,6 +1359,9 @@ func handleOpenResponsesNonStream(c echo.Context, responseID string, createdAt i template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } // Extract reasoning from result before cleaning reasoningContent, cleanedResult := reason.ExtractReasoningComplete(result, thinkingStartToken, cfg.ReasoningConfig) @@ -1640,6 +1643,9 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6 template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } // Track state for streaming var currentMessageID string diff --git a/pkg/reasoning/reasoning.go b/pkg/reasoning/reasoning.go index 108276c3d59b..e6a03288ec88 100644 --- a/pkg/reasoning/reasoning.go +++ b/pkg/reasoning/reasoning.go @@ -21,6 +21,17 @@ import ( // - [THINK] (Magistral models) // Custom tokens from config are checked first, then default tokens. func DetectThinkingStartToken(prompt string, config *Config) string { + return detectThinkingStartToken(prompt, config, true) +} + +// DetectThinkingStartTokenInTemplate detects a possible prefill in an +// unrendered tokenizer template. Marker ordering cannot reveal which Jinja +// branch will render, so matching closing markers are intentionally ignored. +func DetectThinkingStartTokenInTemplate(template string, config *Config) string { + return detectThinkingStartToken(template, config, false) +} + +func detectThinkingStartToken(prompt string, config *Config, honorClosingToken bool) string { // Common thinking start tokens (in order of specificity - longer first) // Based on llama.cpp's chat-parser.cpp implementations defaultTokens := []string{ @@ -44,7 +55,8 @@ func DetectThinkingStartToken(prompt string, config *Config) string { // Check if prompt ends with any of these tokens (allowing for trailing whitespace/newlines) trimmedPrompt := strings.TrimRight(prompt, " \t\n\r") for _, token := range thinkingStartTokens { - if strings.Contains(trimmedPrompt, token) { + if strings.Contains(trimmedPrompt, token) && + (!honorClosingToken || !thinkingTokenClosedAfterLastStart(trimmedPrompt, token, config)) { return token } } @@ -67,6 +79,15 @@ func DetectThinkingStartToken(prompt string, config *Config) string { return "" } +func thinkingTokenClosedAfterLastStart(prompt, startToken string, config *Config) bool { + endToken := ClosingTokenForStart(startToken, config) + if endToken == "" { + return false + } + + return strings.LastIndex(prompt, endToken) > strings.LastIndex(prompt, startToken) +} + // ExtractReasoningWithConfig extracts reasoning from content with the given config. // If reasoning is disabled, it returns the original content. // If thinking start token prefill is enabled, it prepends the thinking start token to the content. diff --git a/pkg/reasoning/reasoning_test.go b/pkg/reasoning/reasoning_test.go index 5e6151b01b47..c58d34a5900e 100644 --- a/pkg/reasoning/reasoning_test.go +++ b/pkg/reasoning/reasoning_test.go @@ -410,6 +410,29 @@ var _ = Describe("DetectThinkingStartToken", func() { token := DetectThinkingStartToken(prompt, nil) Expect(token).To(Equal("")) }) + + It("should ignore a Gemma thinking token that is already closed in the prompt", func() { + prompt := "<|turn>model\n<|channel>thought\n\n" + token := DetectThinkingStartToken(prompt, nil) + Expect(token).To(BeEmpty()) + + extractor := NewReasoningExtractor(token, Config{}) + reasoningDelta, contentDelta := extractor.ProcessToken("READY.") + Expect(reasoningDelta).To(BeEmpty()) + Expect(contentDelta).To(Equal("READY.")) + }) + + It("should preserve prefill detection for unrendered conditional templates", func() { + template := "{% if enable_thinking %}{% else %}{% endif %}" + token := DetectThinkingStartTokenInTemplate(template, nil) + Expect(token).To(Equal("")) + }) + + It("should ignore user Jinja text before a preclosed Gemma prompt suffix", func() { + prompt := "Explain {{ variable }}\n<|turn>model\n<|channel>thought\n\n" + token := DetectThinkingStartToken(prompt, nil) + Expect(token).To(BeEmpty()) + }) }) Context("when prompt does not contain thinking tokens", func() { From 28939644bce2680ab162a508dbf0a1728a260f7e Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:04:10 +0000 Subject: [PATCH 2/2] fix(gallery): use tokenizer templates for Gemma Let the model-provided tokenizer template format Gemma conversations instead of maintaining a shared inline prompt template. Assisted-by: Codex:gpt-5 --- gallery/gemma.yaml | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/gallery/gemma.yaml b/gallery/gemma.yaml index 1e08fed813a0..c86b9d91b51b 100644 --- a/gallery/gemma.yaml +++ b/gallery/gemma.yaml @@ -10,33 +10,5 @@ config_file: | - - template: - chat: | - {{.Input }} - model - chat_message: |- - {{if eq .RoleName "assistant" }}model{{else}}{{ .RoleName }}{{end}} - {{ if .FunctionCall -}} - {{ else if eq .RoleName "tool" -}} - {{ end -}} - {{ if .Content -}} - {{.Content -}} - {{ end -}} - {{ if .FunctionCall -}} - {{toJson .FunctionCall}} - {{ end -}} - completion: | - {{.Input}} - function: | - system - You have access to functions. If you decide to invoke any of the function(s), - you MUST put it in the format of - {"name": function name, "parameters": dictionary of argument name and its value} - - You SHOULD NOT include any other text in the response if you call a function - {{range .Functions}} - {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }} - {{end}} - - {{.Input -}} - model + use_tokenizer_template: true name: gemma