[0828] 插件 I/O 序列化器迁移到 UTF-8 raw

TeXmacs/plugins/autosave/progs/init-autosave.scm

@@ -19,7 +19,7 @@
   (with u
     (pre-serialize lan t)
     (with s
-      (texmacs->code (stree->tree u) "SourceCode")
+      (texmacs->utf8raw (stree->tree u))
       (string-append s "\n<EOF>\n")
     ) ;with
   ) ;with

TeXmacs/plugins/gnuplot/progs/init-gnuplot.scm

@@ -20,7 +20,7 @@
     (lazy-format (data gnuplot) gnuplot)
 
     (define (gnuplot-serialize lan t)
-      (let* ((u (pre-serialize lan t)) (s (texmacs->code (stree->tree u) "utf-8")))
+      (let* ((u (pre-serialize lan t)) (s (texmacs->utf8raw (stree->tree u))))
         (string-append s "\n<EOF>\n")
       ) ;let*
     ) ;define

TeXmacs/plugins/goldfish/progs/init-goldfish.scm

@@ -16,7 +16,7 @@
 (lazy-format (data r7rs) r7rs)
 
 (define (goldfish-serialize lan t)
-  (let* ((u (pre-serialize lan t)) (s (texmacs->code (stree->tree u) "SourceCode")))
+  (let* ((u (pre-serialize lan t)) (s (texmacs->utf8raw (stree->tree u))))
     (string-append s "\n<EOF>\n")
   ) ;let*
 ) ;define

TeXmacs/plugins/julia/progs/init-julia.scm

@@ -15,7 +15,7 @@
 
 (define (julia-serialize lan t)
   (let* ((u (pre-serialize lan t))
-         (s (texmacs->code (stree->tree u) "SourceCode")))
+         (s (texmacs->utf8raw (stree->tree u))))
     (string-append s "\n<EOF>\n")))
 
 (define (julia-entry)

TeXmacs/plugins/maxima/progs/init-maxima.scm

@@ -14,7 +14,7 @@
 (use-modules (binary maxima))
 
 (define (maxima-serialize lan t)
-  (with s (string-drop-right (verbatim-serialize lan t) 1)
+  (with s (string-drop-right (utf8raw-serialize lan t) 1)
     (cond ((== s "") "0;\n")
           ((in? (string-ref s (- (string-length s) 1)) '(#\; #\$))
            (string-append s "\n"))

TeXmacs/plugins/python/progs/init-python.scm

@@ -34,7 +34,7 @@
 
 (define (python-serialize lan t)
   (with u (pre-serialize lan t)
-    (with s (texmacs->code (stree->tree u) "SourceCode")
+    (with s (texmacs->utf8raw (stree->tree u))
       (string-append  s  "\n<EOF>\n"))))
 
 (define (python-utf8-command)

TeXmacs/plugins/quiver/progs/init-quiver.scm

@@ -18,7 +18,7 @@
 
     (define (quiver-serialize lan t)
       (let* ((u (pre-serialize lan t))
-             (s (texmacs->code (stree->tree u) "SourceCode")))
+             (s (texmacs->utf8raw (stree->tree u))))
         (string-append s "\n<EOF>\n")
       )
     )

TeXmacs/plugins/tikz/progs/init-tikz.scm

@@ -20,7 +20,7 @@
 
     (define (tikz-serialize lan t)
       (let* ((u (pre-serialize lan t))
-             (s (texmacs->code (stree->tree u) "SourceCode")))
+             (s (texmacs->utf8raw (stree->tree u))))
         (string-append s "\n<EOF>\n")
       )
     )

TeXmacs/progs/convert/rewrite/init-rewrite.scm

@@ -77,6 +77,16 @@
 
 (converter code-snippet texmacs-tree (:function code-snippet->texmacs))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; UTF-8 raw (plugin I/O)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(tm-define (texmacs->utf8raw t)
+  (if (tree? t) (cpp-tree->utf8raw t) (texmacs->utf8raw (tm->tree t)))
+) ;tm-define
+
+(tm-define (utf8raw->texmacs s) (cpp-utf8raw->tree s))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Verbatim
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

TeXmacs/progs/dynamic/program-edit.scm

@@ -742,7 +742,8 @@
                 (opts (input-options t))
                 (st (tree->stree (tree-ref t 1)))
                 (pre (plugin-preprocess lan ses st opts))
-                (in (plugin-serialize lan pre))
+                (pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree pre))))
+                (in (plugin-serialize lan pre-u8))
                 (rew (if (string-ends? in "\n") (string-drop-right in 1) in))
                 (cmd (string-append "(input-done? " (string-quote rew) ")"))
                 (ret (lambda (done?) (kbd-enter-sub t done?)))

TeXmacs/progs/dynamic/session-edit.scm

@@ -1011,7 +1011,8 @@
                 (opts (input-options t))
                 (st (tree->stree (tree-ref t 1)))
                 (pre (plugin-preprocess lan ses st opts))
-                (in (plugin-serialize lan pre))
+                (pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree pre))))
+                (in (plugin-serialize lan pre-u8))
                 (rew (if (string-ends? in "\n") (string-drop-right in 1) in))
                 (cmd (string-append "(input-done? " (string-quote rew) ")"))
                 (ret (lambda (done?) (kbd-enter-sub t done?)))

TeXmacs/progs/init-research.scm

@@ -129,7 +129,7 @@
 (lazy-define (utils library cursor) notify-cursor-moved)
 (lazy-define (utils edit variants) make-inline-tag-list make-wrapped-tag-list)
 (lazy-define (utils cas cas-out) cas->stree)
-(lazy-define (utils plugins plugin-cmd) pre-serialize verbatim-serialize)
+(lazy-define (utils plugins plugin-cmd) pre-serialize utf8raw-serialize)
 (lazy-define (utils test test-convert)
   delayed-quit
   build-manual
@@ -534,7 +534,7 @@
   export-selection-as-graphics
   clipboard-copy-image
 ) ;lazy-define
-(lazy-define (convert rewrite init-rewrite) texmacs->code texmacs->verbatim)
+(lazy-define (convert rewrite init-rewrite) texmacs->code texmacs->verbatim texmacs->utf8raw utf8raw->texmacs)
 (lazy-define (convert html tmhtml) ext-tmhtml-eqnarray*)
 (define-secure-symbols ext-tmhtml-eqnarray*)
 (lazy-define (convert html tmhtml-expand) tmhtml-env-patch)

TeXmacs/progs/utils/plugins/plugin-cmd.scm

@@ -28,14 +28,14 @@
   ) ;cond
 ) ;tm-define
 
-(define (hacked-texmacs->code x)
-  (with r (texmacs->code x "utf-8") (string-replace r "`" "`"))
+(define (hacked-texmacs->utf8raw x)
+  (with r (texmacs->utf8raw x) (string-replace r "`" "`"))
 ) ;define
 
-(tm-define (verbatim-serialize lan t)
+(tm-define (utf8raw-serialize lan t)
   (with u
     (pre-serialize lan t)
-    (string-append (escape-verbatim (hacked-texmacs->code u)) "\n")
+    (string-append (escape-verbatim (hacked-texmacs->utf8raw u)) "\n")
   ) ;with
 ) ;tm-define
 
@@ -44,7 +44,7 @@
     (pre-serialize lan t)
     (string-append (char->string #\x02)
       "utf8:"
-      (escape-generic (texmacs->code u "utf-8"))
+      (escape-generic (texmacs->utf8raw u))
       (char->string #\x05)
     ) ;string-append
   ) ;with
@@ -53,7 +53,7 @@
 (tm-define (plugin-serialize lan t)
   (with fun
     (ahash-ref plugin-serializer lan)
-    (if fun (fun lan t) (verbatim-serialize lan t))
+    (if fun (fun lan t) (utf8raw-serialize lan t))
   ) ;with
 ) ;tm-define
 

TeXmacs/tests/0828.scm

@@ -0,0 +1,222 @@
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; MODULE      : 0828.scm
+;; DESCRIPTION : UTF-8 raw 插件 I/O 序列化器测试
+;; COPYRIGHT   : (C) 2026 AcceleratorX
+;;
+;; This software falls under the GNU general public license version 3 or later.
+;; It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE
+;; in the root directory or <http://www.gnu.org/licenses/gpl-3.0.html>.
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(import (scheme base)
+        (liii check)
+        (liii string))
+
+(check-set-mode! 'report-failed)
+
+;;; ========== 辅助函数 ==========
+
+;; 由含 UTF-8 字符串标签的 stree 直接构造 UTF-8 tree
+(define (utf8-tree label)
+  (stree->tree `(document ,label)))
+
+;; 加载插件 init 脚本（注册其自定义 serializer）
+(define (load-plugin-init name)
+  (load (string-append (url->system (get-texmacs-path))
+                       "/plugins/" name "/progs/init-" name ".scm")))
+
+;; 用 UTF-8 stree 调 plugin-serialize（plugin-serialize 约定接收 stree）
+(define (plugin-serialize-utf8 lan label)
+  (plugin-serialize lan `(document ,label)))
+
+;;; ========== texmacs->utf8raw / utf8raw->texmacs 往返 ==========
+
+(define (test-utf8raw-roundtrip)
+  (define (roundtrip s)
+    (texmacs->utf8raw (utf8raw->texmacs s)))
+  (check (roundtrip "中文测试") => "中文测试")
+  (check (roundtrip "αβγδ") => "αβγδ")
+  (check (roundtrip "Hello 世界") => "Hello 世界")
+  (check (roundtrip "∑∏∫√") => "∑∏∫√")
+  (check (roundtrip "🎉🎊") => "🎉🎊")
+  (check (roundtrip "line1\nline2") => "line1\nline2")
+  (check (roundtrip "a\n\nb") => "a\n\nb")
+  (check (roundtrip "") => "")
+  (check (roundtrip "  ") => "  "))
+
+;;; ========== 由 UTF-8 tree 序列化 ==========
+
+(define (test-utf8raw-from-tree)
+  (check (texmacs->utf8raw (utf8-tree "中文测试")) => "中文测试")
+  (check (texmacs->utf8raw (utf8-tree "αβγδ")) => "αβγδ")
+  (check (texmacs->utf8raw (utf8-tree "Hello 世界")) => "Hello 世界")
+  (check (texmacs->utf8raw (utf8-tree "∑∏∫√")) => "∑∏∫√")
+  (check (texmacs->utf8raw (stree->tree '(document "line1" "line2"))) => "line1\nline2"))
+
+;;; ========== utf8raw->texmacs 保留原始字节 ==========
+
+(define (test-utf8raw-to-tree)
+  ;; utf8: 块里的 <#XXXX> 字面必须原样保留，不得被解码
+  (define t (utf8raw->texmacs "<#5206><#5B50>"))
+  (check (texmacs->utf8raw t) => "<#5206><#5B50>")
+  ;; CR/LF 归一化
+  (check (texmacs->utf8raw (utf8raw->texmacs "a\r\nb")) => "a\nb")
+  (check (texmacs->utf8raw (utf8raw->texmacs "a\rb")) => "a\nb"))
+
+;;; ========== 默认 serializer 输出 UTF-8 raw ==========
+
+(define (test-utf8raw-serialize-utf8)
+  (define s (utf8raw-serialize "python" (utf8-tree "print('中文')")))
+  (check (string-contains? s "中文") => #t)
+  (check (string-contains? s "<#4E2D>") => #f))
+
+(define (test-generic-serialize-utf8)
+  (define s (generic-serialize "python" (utf8-tree "print('中文')")))
+  (check (string-starts? s (char->string #\x02)) => #t)
+  (check (string-contains? s "utf8:") => #t)
+  (check (string-contains? s "中文") => #t)
+  (check (string-contains? s "<#4E2D>") => #f)
+  (check (string-ends? s (char->string #\x05)) => #t))
+
+;;; ========== 各插件自定义 serializer 输出 UTF-8 ==========
+
+(define (test-python-serialize-utf8)
+  (load-plugin-init "python")
+  (with s (plugin-serialize-utf8 "python" "print('中文')")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-julia-serialize-utf8)
+  (load-plugin-init "julia")
+  (with s (plugin-serialize-utf8 "julia" "println(\"中文\")")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-goldfish-serialize-utf8)
+  (load-plugin-init "goldfish")
+  (with s (plugin-serialize-utf8 "goldfish" "(display \"中文\")")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-gnuplot-serialize-utf8)
+  (load-plugin-init "gnuplot")
+  (with s (plugin-serialize-utf8 "gnuplot" "set title \"中文\"")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-autosave-serialize-utf8)
+  (load-plugin-init "autosave")
+  (with s (plugin-serialize-utf8 "autosave" "(display \"中文\")")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-tikz-serialize-utf8)
+  (load-plugin-init "tikz")
+  (with s (plugin-serialize-utf8 "tikz" "\\node {中文};")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+(define (test-quiver-serialize-utf8)
+  (load-plugin-init "quiver")
+  (with s (plugin-serialize-utf8 "quiver" "\\node {中文};")
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (when (string-contains? s "<EOF>")
+      (check (string-ends? s "\n<EOF>\n") => #t))))
+
+;; Maxima 经 (plugin-configure :serializer ...) 注册，依赖二进制存在；测试环境无二进制，
+;; plugin-serialize 会落回默认 utf8raw-serialize 而测不到 maxima-serialize，故直接调用。
+(define (test-maxima-serialize-utf8)
+  (load-plugin-init "maxima")
+  (with s (maxima-serialize "maxima" '(document "x:中文"))
+    (check (string-contains? s "中文") => #t)
+    (check (string-contains? s "<#4E2D>") => #f)
+    (check (string-ends? s ";\n") => #t))
+  (check (maxima-serialize "maxima" '(document "")) => "0;\n")
+  (with s (maxima-serialize "maxima" '(document "中文$"))
+    (check (string-ends? s "中文$\n") => #t)))
+
+;;; ========== 输入端：utf8raw->texmacs 后转 Herk ==========
+
+;; 模拟 input.cpp：把 utf8: 负载解析成 UTF-8 tree，再转成内部 Herk tree。
+(define (test-input-utf8-to-herk)
+  (define herk-str (texmacs->stm (utf8-tree->herk-tree (utf8raw->texmacs "print('中文')"))))
+  (check (string-contains? herk-str "<#4E2D>") => #t)
+  (check (string-contains? herk-str "<#6587>") => #t)
+  (check (string-contains? herk-str "中文") => #f))
+
+;; emoji（BMP 外）在 utf8_to_herk 映射表里没有对应项，tree_utf8_to_herk 须以 <#XXXX>
+;; 保留而非丢弃——这是 0828 相对 0827（走 utf8_to_cork 会静默丢失此类字符）的关键修正。
+(define (test-input-emoji-to-herk)
+  (define herk-str (texmacs->stm (utf8-tree->herk-tree (utf8raw->texmacs "x:🎉"))))
+  (check (string-contains? herk-str "<#1F389>") => #t)
+  (check (string-contains? herk-str "🎉") => #f))
+
+;; 普通文本与 <#XXXX> 字面混合：<#XXXX> 原样存活（不被当作 UTF-8 字节再解码），CJK 正常往返。
+(define (test-utf8raw-mixed-hex-literal)
+  (define t (utf8raw->texmacs "代码<#5206>"))
+  (check (texmacs->utf8raw t) => "代码<#5206>"))
+
+;;; ========== 写出端：Herk tree -> UTF-8 tree -> raw 串 ==========
+
+;; 模拟主进程到插件的路径：内部 Herk tree 先转 UTF-8 tree，再由 texmacs->utf8raw 序列化。
+(define (test-write-path-herk-to-utf8raw)
+  (define s (texmacs->utf8raw
+              (herk-tree->utf8-tree
+                (stm-snippet->texmacs "(frac \"<#5206><#5B50>\" \"<#5206><#6BCD>\")"))))
+  (check (string-contains? s "分子") => #t)
+  (check (string-contains? s "分母") => #t)
+  (check (string-contains? s "<#5206>") => #f))
+
+;;; ========== input-done? 旁路：序列化前 Herk stree 转 UTF-8 ==========
+
+;; session-edit.scm / program-edit.scm 在 input-done? 路径上，先把 pre（Herk stree）转成
+;; UTF-8 stree 再 plugin-serialize。下方正向断言验证转换后输出 UTF-8；反向断言证明不转换
+;; 则 Herk 的 <#XXXX> 字面会原样漏给插件（插件收到 8 个 ASCII 字符 "<#4E2D>" 而非"中"），
+;; 即该转换是必需的、不可省略。
+(define (test-input-done-herk-to-utf8)
+  (load-plugin-init "python")
+  (define herk-stree '(document "print('<#4E2D><#6587>')"))
+  (define pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree herk-stree))))
+  (check (string-contains? (plugin-serialize "python" pre-u8) "中文") => #t)
+  (check (string-contains? (plugin-serialize "python" pre-u8) "<#4E2D>") => #f)
+  (define bad (plugin-serialize "python" herk-stree))
+  (check (string-contains? bad "<#4E2D>") => #t)
+  (check (string-contains? bad "中文") => #f))
+
+;;; ========== 测试入口 ==========
+
+(tm-define (test_0828)
+  (test-utf8raw-roundtrip)
+  (test-utf8raw-from-tree)
+  (test-utf8raw-to-tree)
+  (test-utf8raw-serialize-utf8)
+  (test-generic-serialize-utf8)
+  (test-python-serialize-utf8)
+  (test-julia-serialize-utf8)
+  (test-goldfish-serialize-utf8)
+  (test-gnuplot-serialize-utf8)
+  (test-autosave-serialize-utf8)
+  (test-tikz-serialize-utf8)
+  (test-quiver-serialize-utf8)
+  (test-maxima-serialize-utf8)
+  (test-input-utf8-to-herk)
+  (test-input-emoji-to-herk)
+  (test-utf8raw-mixed-hex-literal)
+  (test-write-path-herk-to-utf8raw)
+  (test-input-done-herk-to-utf8)
+  (check-report))