fix(reasoning): keep preclosed Gemma streams in content

core/http/endpoints/openai/chat.go

@@ -696,6 +696,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 					template = predInput
 				}
 				thinkingStartToken := reason.DetectThinkingStartToken(template, &config.ReasoningConfig)
+				if config.TemplateConfig.UseTokenizerTemplate {
+					thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig)
+				}
 
 				xlog.Debug("Thinking start token", "thinkingStartToken", thinkingStartToken, "template", template)
 

core/http/endpoints/openai/chat_stream_workers.go

@@ -150,6 +150,9 @@ func processStream(
 		template = s
 	}
 	thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig)
+	if cfg.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig)
+	}
 	extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig)
 
 	// preferAutoparser is sticky: once the C++ autoparser has ever classified
@@ -248,6 +251,9 @@ func processStreamWithTools(
 		template = prompt
 	}
 	thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig)
+	if cfg.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig)
+	}
 	extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig)
 
 	result := ""

core/http/endpoints/openai/realtime.go

@@ -2289,6 +2289,9 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa
 		template = config.TemplateConfig.Chat
 	}
 	thinkingStartToken := reasoning.DetectThinkingStartToken(template, &config.ReasoningConfig)
+	if config.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig)
+	}
 
 	// When the C++ autoparser emitted ChatDeltas with actionable data,
 	// prefer them — the backend clears Reply.Message in that path and

core/http/endpoints/openai/realtime_stream.go

@@ -145,6 +145,9 @@ func streamLLMResponse(ctx context.Context, session *Session, conv *Conversation
 		template = llmCfg.TemplateConfig.Chat
 	}
 	thinkingStartToken := reasoning.DetectThinkingStartToken(template, &llmCfg.ReasoningConfig)
+	if llmCfg.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &llmCfg.ReasoningConfig)
+	}
 
 	// The autoparser (tokenizer-template path) already delivers reasoning-free
 	// content. Prefilling the thinking start token here would re-tag that clean

core/http/endpoints/openresponses/responses.go

@@ -1359,6 +1359,9 @@ func handleOpenResponsesNonStream(c echo.Context, responseID string, createdAt i
 		template = predInput
 	}
 	thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig)
+	if cfg.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig)
+	}
 
 	// Extract reasoning from result before cleaning
 	reasoningContent, cleanedResult := reason.ExtractReasoningComplete(result, thinkingStartToken, cfg.ReasoningConfig)
@@ -1640,6 +1643,9 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
 		template = predInput
 	}
 	thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig)
+	if cfg.TemplateConfig.UseTokenizerTemplate {
+		thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig)
+	}
 
 	// Track state for streaming
 	var currentMessageID string

gallery/gemma.yaml

@@ -10,33 +10,5 @@ config_file: |
         - <end_of_turn>
         - <start_of_turn>
     template:
-        chat: |
-            {{.Input }}
-            <start_of_turn>model
-        chat_message: |-
-            <start_of_turn>{{if eq .RoleName "assistant" }}model{{else}}{{ .RoleName }}{{end}}
-            {{ if .FunctionCall -}}
-            {{ else if eq .RoleName "tool" -}}
-            {{ end -}}
-            {{ if .Content -}}
-            {{.Content -}}
-            {{ end -}}
-            {{ if .FunctionCall -}}
-            {{toJson .FunctionCall}}
-            {{ end -}}<end_of_turn>
-        completion: |
-            {{.Input}}
-        function: |
-            <start_of_turn>system
-            You have access to functions. If you decide to invoke any of the function(s),
-            you MUST put it in the format of
-            {"name": function name, "parameters": dictionary of argument name and its value}
-
-            You SHOULD NOT include any other text in the response if you call a function
-            {{range .Functions}}
-            {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
-            {{end}}
-            <end_of_turn>
-            {{.Input -}}
-            <start_of_turn>model
+        use_tokenizer_template: true
 name: gemma

pkg/reasoning/reasoning.go

@@ -21,6 +21,17 @@ import (
 // - [THINK]                 (Magistral models)
 // Custom tokens from config are checked first, then default tokens.
 func DetectThinkingStartToken(prompt string, config *Config) string {
+	return detectThinkingStartToken(prompt, config, true)
+}
+
+// DetectThinkingStartTokenInTemplate detects a possible prefill in an
+// unrendered tokenizer template. Marker ordering cannot reveal which Jinja
+// branch will render, so matching closing markers are intentionally ignored.
+func DetectThinkingStartTokenInTemplate(template string, config *Config) string {
+	return detectThinkingStartToken(template, config, false)
+}
+
+func detectThinkingStartToken(prompt string, config *Config, honorClosingToken bool) string {
 	// Common thinking start tokens (in order of specificity - longer first)
 	// Based on llama.cpp's chat-parser.cpp implementations
 	defaultTokens := []string{
@@ -44,7 +55,8 @@ func DetectThinkingStartToken(prompt string, config *Config) string {
 	// Check if prompt ends with any of these tokens (allowing for trailing whitespace/newlines)
 	trimmedPrompt := strings.TrimRight(prompt, " \t\n\r")
 	for _, token := range thinkingStartTokens {
-		if strings.Contains(trimmedPrompt, token) {
+		if strings.Contains(trimmedPrompt, token) &&
+			(!honorClosingToken || !thinkingTokenClosedAfterLastStart(trimmedPrompt, token, config)) {
 			return token
 		}
 	}
@@ -67,6 +79,15 @@ func DetectThinkingStartToken(prompt string, config *Config) string {
 	return ""
 }
 
+func thinkingTokenClosedAfterLastStart(prompt, startToken string, config *Config) bool {
+	endToken := ClosingTokenForStart(startToken, config)
+	if endToken == "" {
+		return false
+	}
+
+	return strings.LastIndex(prompt, endToken) > strings.LastIndex(prompt, startToken)
+}
+
 // ExtractReasoningWithConfig extracts reasoning from content with the given config.
 // If reasoning is disabled, it returns the original content.
 // If thinking start token prefill is enabled, it prepends the thinking start token to the content.

pkg/reasoning/reasoning_test.go

@@ -410,6 +410,29 @@ var _ = Describe("DetectThinkingStartToken", func() {
 			token := DetectThinkingStartToken(prompt, nil)
 			Expect(token).To(Equal("<think>"))
 		})
+
+		It("should ignore a Gemma thinking token that is already closed in the prompt", func() {
+			prompt := "<|turn>model\n<|channel>thought\n<channel|>\n"
+			token := DetectThinkingStartToken(prompt, nil)
+			Expect(token).To(BeEmpty())
+
+			extractor := NewReasoningExtractor(token, Config{})
+			reasoningDelta, contentDelta := extractor.ProcessToken("READY.")
+			Expect(reasoningDelta).To(BeEmpty())
+			Expect(contentDelta).To(Equal("READY."))
+		})
+
+		It("should preserve prefill detection for unrendered conditional templates", func() {
+			template := "{% if enable_thinking %}<think>{% else %}<think></think>{% endif %}"
+			token := DetectThinkingStartTokenInTemplate(template, nil)
+			Expect(token).To(Equal("<think>"))
+		})
+
+		It("should ignore user Jinja text before a preclosed Gemma prompt suffix", func() {
+			prompt := "Explain {{ variable }}\n<|turn>model\n<|channel>thought\n<channel|>\n"
+			token := DetectThinkingStartToken(prompt, nil)
+			Expect(token).To(BeEmpty())
+		})
 	})
 
 	Context("when prompt does not contain thinking tokens", func() {