From 53885de682a703b67e2067df71f4bde2b00cf5fb Mon Sep 17 00:00:00 2001 From: cscs <35064549+zpczpc@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:52:27 +0800 Subject: [PATCH] Document that BpeOptions.ByteLevel requires a PreTokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The documentation for `BpeOptions.ByteLevel` only described the byte <-> unicode mapping (for example `Space -> 'Ġ'`) and did not mention that a pre-tokenizer is required. Without a pre-tokenizer, byte-level encoding maps the space character to `'Ġ'` but does not keep the whitespace attached to the following token, so spaces and newlines are dropped during encoding and cannot be recovered by decoding. This adds a `` block stating the requirement and points at `RegexPreTokenizer` with the GPT-2 pattern as the configuration that keeps the round trip lossless. Relates to #7715. --- src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs index 8eee50ac66..1c6edd7eac 100644 --- a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs +++ b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs @@ -143,6 +143,14 @@ public BpeOptions(string vocabFile, string? mergesFile = null) /// if true, the input text will be converted to UTF-8 bytes before encoding it. /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character). /// + /// + /// Byte-level encoding is normally paired with a byte-level pre-tokenizer. When this property is set to + /// , must be set as well. + /// On its own, byte-level encoding maps the space character to 'Ġ' but does not keep the whitespace attached + /// to the following token, so spaces and newlines are dropped during encoding and cannot be recovered by decoding. + /// Configuring a byte-level pre-tokenizer, for example built from the GPT-2 + /// pattern, makes the round trip lossless. + /// public bool ByteLevel { get; set; } ///