diff --git a/TeXmacs/plugins/autosave/progs/init-autosave.scm b/TeXmacs/plugins/autosave/progs/init-autosave.scm index 22767fd809..599e53ec19 100644 --- a/TeXmacs/plugins/autosave/progs/init-autosave.scm +++ b/TeXmacs/plugins/autosave/progs/init-autosave.scm @@ -19,7 +19,7 @@ (with u (pre-serialize lan t) (with s - (texmacs->code (stree->tree u) "SourceCode") + (texmacs->utf8raw (stree->tree u)) (string-append s "\n\n") ) ;with ) ;with diff --git a/TeXmacs/plugins/gnuplot/progs/init-gnuplot.scm b/TeXmacs/plugins/gnuplot/progs/init-gnuplot.scm index 758e5aa75b..dbe0b5278e 100644 --- a/TeXmacs/plugins/gnuplot/progs/init-gnuplot.scm +++ b/TeXmacs/plugins/gnuplot/progs/init-gnuplot.scm @@ -20,7 +20,7 @@ (lazy-format (data gnuplot) gnuplot) (define (gnuplot-serialize lan t) - (let* ((u (pre-serialize lan t)) (s (texmacs->code (stree->tree u) "utf-8"))) + (let* ((u (pre-serialize lan t)) (s (texmacs->utf8raw (stree->tree u)))) (string-append s "\n\n") ) ;let* ) ;define diff --git a/TeXmacs/plugins/goldfish/progs/init-goldfish.scm b/TeXmacs/plugins/goldfish/progs/init-goldfish.scm index 9b1d83020c..3af2836e0b 100644 --- a/TeXmacs/plugins/goldfish/progs/init-goldfish.scm +++ b/TeXmacs/plugins/goldfish/progs/init-goldfish.scm @@ -16,7 +16,7 @@ (lazy-format (data r7rs) r7rs) (define (goldfish-serialize lan t) - (let* ((u (pre-serialize lan t)) (s (texmacs->code (stree->tree u) "SourceCode"))) + (let* ((u (pre-serialize lan t)) (s (texmacs->utf8raw (stree->tree u)))) (string-append s "\n\n") ) ;let* ) ;define diff --git a/TeXmacs/plugins/julia/progs/init-julia.scm b/TeXmacs/plugins/julia/progs/init-julia.scm index 8acecfd947..e59d316063 100644 --- a/TeXmacs/plugins/julia/progs/init-julia.scm +++ b/TeXmacs/plugins/julia/progs/init-julia.scm @@ -15,7 +15,7 @@ (define (julia-serialize lan t) (let* ((u (pre-serialize lan t)) - (s (texmacs->code (stree->tree u) "SourceCode"))) + (s (texmacs->utf8raw (stree->tree u)))) (string-append s "\n\n"))) (define (julia-entry) diff --git a/TeXmacs/plugins/maxima/progs/init-maxima.scm b/TeXmacs/plugins/maxima/progs/init-maxima.scm index e1be8d90fb..6e1fc79304 100644 --- a/TeXmacs/plugins/maxima/progs/init-maxima.scm +++ b/TeXmacs/plugins/maxima/progs/init-maxima.scm @@ -14,7 +14,7 @@ (use-modules (binary maxima)) (define (maxima-serialize lan t) - (with s (string-drop-right (verbatim-serialize lan t) 1) + (with s (string-drop-right (utf8raw-serialize lan t) 1) (cond ((== s "") "0;\n") ((in? (string-ref s (- (string-length s) 1)) '(#\; #\$)) (string-append s "\n")) diff --git a/TeXmacs/plugins/python/progs/init-python.scm b/TeXmacs/plugins/python/progs/init-python.scm index 21db55ef0b..6aa336ade3 100644 --- a/TeXmacs/plugins/python/progs/init-python.scm +++ b/TeXmacs/plugins/python/progs/init-python.scm @@ -34,7 +34,7 @@ (define (python-serialize lan t) (with u (pre-serialize lan t) - (with s (texmacs->code (stree->tree u) "SourceCode") + (with s (texmacs->utf8raw (stree->tree u)) (string-append s "\n\n")))) (define (python-utf8-command) diff --git a/TeXmacs/plugins/quiver/progs/init-quiver.scm b/TeXmacs/plugins/quiver/progs/init-quiver.scm index ca9618ddb2..3ef2db3b9a 100644 --- a/TeXmacs/plugins/quiver/progs/init-quiver.scm +++ b/TeXmacs/plugins/quiver/progs/init-quiver.scm @@ -18,7 +18,7 @@ (define (quiver-serialize lan t) (let* ((u (pre-serialize lan t)) - (s (texmacs->code (stree->tree u) "SourceCode"))) + (s (texmacs->utf8raw (stree->tree u)))) (string-append s "\n\n") ) ) diff --git a/TeXmacs/plugins/tikz/progs/init-tikz.scm b/TeXmacs/plugins/tikz/progs/init-tikz.scm index 13909100d7..8f96af1d8f 100644 --- a/TeXmacs/plugins/tikz/progs/init-tikz.scm +++ b/TeXmacs/plugins/tikz/progs/init-tikz.scm @@ -20,7 +20,7 @@ (define (tikz-serialize lan t) (let* ((u (pre-serialize lan t)) - (s (texmacs->code (stree->tree u) "SourceCode"))) + (s (texmacs->utf8raw (stree->tree u)))) (string-append s "\n\n") ) ) diff --git a/TeXmacs/progs/convert/rewrite/init-rewrite.scm b/TeXmacs/progs/convert/rewrite/init-rewrite.scm index 355e1851c4..1c420ce385 100644 --- a/TeXmacs/progs/convert/rewrite/init-rewrite.scm +++ b/TeXmacs/progs/convert/rewrite/init-rewrite.scm @@ -77,6 +77,16 @@ (converter code-snippet texmacs-tree (:function code-snippet->texmacs)) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; UTF-8 raw (plugin I/O) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(tm-define (texmacs->utf8raw t) + (if (tree? t) (cpp-tree->utf8raw t) (texmacs->utf8raw (tm->tree t))) +) ;tm-define + +(tm-define (utf8raw->texmacs s) (cpp-utf8raw->tree s)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Verbatim ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/TeXmacs/progs/dynamic/program-edit.scm b/TeXmacs/progs/dynamic/program-edit.scm index f03c8a523b..4496cb1251 100644 --- a/TeXmacs/progs/dynamic/program-edit.scm +++ b/TeXmacs/progs/dynamic/program-edit.scm @@ -742,7 +742,8 @@ (opts (input-options t)) (st (tree->stree (tree-ref t 1))) (pre (plugin-preprocess lan ses st opts)) - (in (plugin-serialize lan pre)) + (pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree pre)))) + (in (plugin-serialize lan pre-u8)) (rew (if (string-ends? in "\n") (string-drop-right in 1) in)) (cmd (string-append "(input-done? " (string-quote rew) ")")) (ret (lambda (done?) (kbd-enter-sub t done?))) diff --git a/TeXmacs/progs/dynamic/session-edit.scm b/TeXmacs/progs/dynamic/session-edit.scm index 6e1d5f4147..23cb81e0af 100644 --- a/TeXmacs/progs/dynamic/session-edit.scm +++ b/TeXmacs/progs/dynamic/session-edit.scm @@ -1011,7 +1011,8 @@ (opts (input-options t)) (st (tree->stree (tree-ref t 1))) (pre (plugin-preprocess lan ses st opts)) - (in (plugin-serialize lan pre)) + (pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree pre)))) + (in (plugin-serialize lan pre-u8)) (rew (if (string-ends? in "\n") (string-drop-right in 1) in)) (cmd (string-append "(input-done? " (string-quote rew) ")")) (ret (lambda (done?) (kbd-enter-sub t done?))) diff --git a/TeXmacs/progs/init-research.scm b/TeXmacs/progs/init-research.scm index 4ed3a316ba..72335338ab 100644 --- a/TeXmacs/progs/init-research.scm +++ b/TeXmacs/progs/init-research.scm @@ -129,7 +129,7 @@ (lazy-define (utils library cursor) notify-cursor-moved) (lazy-define (utils edit variants) make-inline-tag-list make-wrapped-tag-list) (lazy-define (utils cas cas-out) cas->stree) -(lazy-define (utils plugins plugin-cmd) pre-serialize verbatim-serialize) +(lazy-define (utils plugins plugin-cmd) pre-serialize utf8raw-serialize) (lazy-define (utils test test-convert) delayed-quit build-manual @@ -534,7 +534,7 @@ export-selection-as-graphics clipboard-copy-image ) ;lazy-define -(lazy-define (convert rewrite init-rewrite) texmacs->code texmacs->verbatim) +(lazy-define (convert rewrite init-rewrite) texmacs->code texmacs->verbatim texmacs->utf8raw utf8raw->texmacs) (lazy-define (convert html tmhtml) ext-tmhtml-eqnarray*) (define-secure-symbols ext-tmhtml-eqnarray*) (lazy-define (convert html tmhtml-expand) tmhtml-env-patch) diff --git a/TeXmacs/progs/utils/plugins/plugin-cmd.scm b/TeXmacs/progs/utils/plugins/plugin-cmd.scm index e18274e933..ed5f0aa220 100644 --- a/TeXmacs/progs/utils/plugins/plugin-cmd.scm +++ b/TeXmacs/progs/utils/plugins/plugin-cmd.scm @@ -28,14 +28,14 @@ ) ;cond ) ;tm-define -(define (hacked-texmacs->code x) - (with r (texmacs->code x "utf-8") (string-replace r "`" "`")) +(define (hacked-texmacs->utf8raw x) + (with r (texmacs->utf8raw x) (string-replace r "`" "`")) ) ;define -(tm-define (verbatim-serialize lan t) +(tm-define (utf8raw-serialize lan t) (with u (pre-serialize lan t) - (string-append (escape-verbatim (hacked-texmacs->code u)) "\n") + (string-append (escape-verbatim (hacked-texmacs->utf8raw u)) "\n") ) ;with ) ;tm-define @@ -44,7 +44,7 @@ (pre-serialize lan t) (string-append (char->string #\x02) "utf8:" - (escape-generic (texmacs->code u "utf-8")) + (escape-generic (texmacs->utf8raw u)) (char->string #\x05) ) ;string-append ) ;with @@ -53,7 +53,7 @@ (tm-define (plugin-serialize lan t) (with fun (ahash-ref plugin-serializer lan) - (if fun (fun lan t) (verbatim-serialize lan t)) + (if fun (fun lan t) (utf8raw-serialize lan t)) ) ;with ) ;tm-define diff --git a/TeXmacs/tests/0828.scm b/TeXmacs/tests/0828.scm new file mode 100644 index 0000000000..a374f62b2e --- /dev/null +++ b/TeXmacs/tests/0828.scm @@ -0,0 +1,222 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MODULE : 0828.scm +;; DESCRIPTION : UTF-8 raw 插件 I/O 序列化器测试 +;; COPYRIGHT : (C) 2026 AcceleratorX +;; +;; This software falls under the GNU general public license version 3 or later. +;; It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE +;; in the root directory or . +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(import (scheme base) + (liii check) + (liii string)) + +(check-set-mode! 'report-failed) + +;;; ========== 辅助函数 ========== + +;; 由含 UTF-8 字符串标签的 stree 直接构造 UTF-8 tree +(define (utf8-tree label) + (stree->tree `(document ,label))) + +;; 加载插件 init 脚本(注册其自定义 serializer) +(define (load-plugin-init name) + (load (string-append (url->system (get-texmacs-path)) + "/plugins/" name "/progs/init-" name ".scm"))) + +;; 用 UTF-8 stree 调 plugin-serialize(plugin-serialize 约定接收 stree) +(define (plugin-serialize-utf8 lan label) + (plugin-serialize lan `(document ,label))) + +;;; ========== texmacs->utf8raw / utf8raw->texmacs 往返 ========== + +(define (test-utf8raw-roundtrip) + (define (roundtrip s) + (texmacs->utf8raw (utf8raw->texmacs s))) + (check (roundtrip "中文测试") => "中文测试") + (check (roundtrip "αβγδ") => "αβγδ") + (check (roundtrip "Hello 世界") => "Hello 世界") + (check (roundtrip "∑∏∫√") => "∑∏∫√") + (check (roundtrip "🎉🎊") => "🎉🎊") + (check (roundtrip "line1\nline2") => "line1\nline2") + (check (roundtrip "a\n\nb") => "a\n\nb") + (check (roundtrip "") => "") + (check (roundtrip " ") => " ")) + +;;; ========== 由 UTF-8 tree 序列化 ========== + +(define (test-utf8raw-from-tree) + (check (texmacs->utf8raw (utf8-tree "中文测试")) => "中文测试") + (check (texmacs->utf8raw (utf8-tree "αβγδ")) => "αβγδ") + (check (texmacs->utf8raw (utf8-tree "Hello 世界")) => "Hello 世界") + (check (texmacs->utf8raw (utf8-tree "∑∏∫√")) => "∑∏∫√") + (check (texmacs->utf8raw (stree->tree '(document "line1" "line2"))) => "line1\nline2")) + +;;; ========== utf8raw->texmacs 保留原始字节 ========== + +(define (test-utf8raw-to-tree) + ;; utf8: 块里的 <#XXXX> 字面必须原样保留,不得被解码 + (define t (utf8raw->texmacs "<#5206><#5B50>")) + (check (texmacs->utf8raw t) => "<#5206><#5B50>") + ;; CR/LF 归一化 + (check (texmacs->utf8raw (utf8raw->texmacs "a\r\nb")) => "a\nb") + (check (texmacs->utf8raw (utf8raw->texmacs "a\rb")) => "a\nb")) + +;;; ========== 默认 serializer 输出 UTF-8 raw ========== + +(define (test-utf8raw-serialize-utf8) + (define s (utf8raw-serialize "python" (utf8-tree "print('中文')"))) + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f)) + +(define (test-generic-serialize-utf8) + (define s (generic-serialize "python" (utf8-tree "print('中文')"))) + (check (string-starts? s (char->string #\x02)) => #t) + (check (string-contains? s "utf8:") => #t) + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (check (string-ends? s (char->string #\x05)) => #t)) + +;;; ========== 各插件自定义 serializer 输出 UTF-8 ========== + +(define (test-python-serialize-utf8) + (load-plugin-init "python") + (with s (plugin-serialize-utf8 "python" "print('中文')") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-julia-serialize-utf8) + (load-plugin-init "julia") + (with s (plugin-serialize-utf8 "julia" "println(\"中文\")") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-goldfish-serialize-utf8) + (load-plugin-init "goldfish") + (with s (plugin-serialize-utf8 "goldfish" "(display \"中文\")") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-gnuplot-serialize-utf8) + (load-plugin-init "gnuplot") + (with s (plugin-serialize-utf8 "gnuplot" "set title \"中文\"") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-autosave-serialize-utf8) + (load-plugin-init "autosave") + (with s (plugin-serialize-utf8 "autosave" "(display \"中文\")") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-tikz-serialize-utf8) + (load-plugin-init "tikz") + (with s (plugin-serialize-utf8 "tikz" "\\node {中文};") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +(define (test-quiver-serialize-utf8) + (load-plugin-init "quiver") + (with s (plugin-serialize-utf8 "quiver" "\\node {中文};") + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (when (string-contains? s "") + (check (string-ends? s "\n\n") => #t)))) + +;; Maxima 经 (plugin-configure :serializer ...) 注册,依赖二进制存在;测试环境无二进制, +;; plugin-serialize 会落回默认 utf8raw-serialize 而测不到 maxima-serialize,故直接调用。 +(define (test-maxima-serialize-utf8) + (load-plugin-init "maxima") + (with s (maxima-serialize "maxima" '(document "x:中文")) + (check (string-contains? s "中文") => #t) + (check (string-contains? s "<#4E2D>") => #f) + (check (string-ends? s ";\n") => #t)) + (check (maxima-serialize "maxima" '(document "")) => "0;\n") + (with s (maxima-serialize "maxima" '(document "中文$")) + (check (string-ends? s "中文$\n") => #t))) + +;;; ========== 输入端:utf8raw->texmacs 后转 Herk ========== + +;; 模拟 input.cpp:把 utf8: 负载解析成 UTF-8 tree,再转成内部 Herk tree。 +(define (test-input-utf8-to-herk) + (define herk-str (texmacs->stm (utf8-tree->herk-tree (utf8raw->texmacs "print('中文')")))) + (check (string-contains? herk-str "<#4E2D>") => #t) + (check (string-contains? herk-str "<#6587>") => #t) + (check (string-contains? herk-str "中文") => #f)) + +;; emoji(BMP 外)在 utf8_to_herk 映射表里没有对应项,tree_utf8_to_herk 须以 <#XXXX> +;; 保留而非丢弃——这是 0828 相对 0827(走 utf8_to_cork 会静默丢失此类字符)的关键修正。 +(define (test-input-emoji-to-herk) + (define herk-str (texmacs->stm (utf8-tree->herk-tree (utf8raw->texmacs "x:🎉")))) + (check (string-contains? herk-str "<#1F389>") => #t) + (check (string-contains? herk-str "🎉") => #f)) + +;; 普通文本与 <#XXXX> 字面混合:<#XXXX> 原样存活(不被当作 UTF-8 字节再解码),CJK 正常往返。 +(define (test-utf8raw-mixed-hex-literal) + (define t (utf8raw->texmacs "代码<#5206>")) + (check (texmacs->utf8raw t) => "代码<#5206>")) + +;;; ========== 写出端:Herk tree -> UTF-8 tree -> raw 串 ========== + +;; 模拟主进程到插件的路径:内部 Herk tree 先转 UTF-8 tree,再由 texmacs->utf8raw 序列化。 +(define (test-write-path-herk-to-utf8raw) + (define s (texmacs->utf8raw + (herk-tree->utf8-tree + (stm-snippet->texmacs "(frac \"<#5206><#5B50>\" \"<#5206><#6BCD>\")")))) + (check (string-contains? s "分子") => #t) + (check (string-contains? s "分母") => #t) + (check (string-contains? s "<#5206>") => #f)) + +;;; ========== input-done? 旁路:序列化前 Herk stree 转 UTF-8 ========== + +;; session-edit.scm / program-edit.scm 在 input-done? 路径上,先把 pre(Herk stree)转成 +;; UTF-8 stree 再 plugin-serialize。下方正向断言验证转换后输出 UTF-8;反向断言证明不转换 +;; 则 Herk 的 <#XXXX> 字面会原样漏给插件(插件收到 8 个 ASCII 字符 "<#4E2D>" 而非"中"), +;; 即该转换是必需的、不可省略。 +(define (test-input-done-herk-to-utf8) + (load-plugin-init "python") + (define herk-stree '(document "print('<#4E2D><#6587>')")) + (define pre-u8 (tree->stree (herk-tree->utf8-tree (stree->tree herk-stree)))) + (check (string-contains? (plugin-serialize "python" pre-u8) "中文") => #t) + (check (string-contains? (plugin-serialize "python" pre-u8) "<#4E2D>") => #f) + (define bad (plugin-serialize "python" herk-stree)) + (check (string-contains? bad "<#4E2D>") => #t) + (check (string-contains? bad "中文") => #f)) + +;;; ========== 测试入口 ========== + +(tm-define (test_0828) + (test-utf8raw-roundtrip) + (test-utf8raw-from-tree) + (test-utf8raw-to-tree) + (test-utf8raw-serialize-utf8) + (test-generic-serialize-utf8) + (test-python-serialize-utf8) + (test-julia-serialize-utf8) + (test-goldfish-serialize-utf8) + (test-gnuplot-serialize-utf8) + (test-autosave-serialize-utf8) + (test-tikz-serialize-utf8) + (test-quiver-serialize-utf8) + (test-maxima-serialize-utf8) + (test-input-utf8-to-herk) + (test-input-emoji-to-herk) + (test-utf8raw-mixed-hex-literal) + (test-write-path-herk-to-utf8raw) + (test-input-done-herk-to-utf8) + (check-report)) diff --git a/devel/0828.md b/devel/0828.md new file mode 100644 index 0000000000..6d8314e2c9 --- /dev/null +++ b/devel/0828.md @@ -0,0 +1,56 @@ +# [0828] 插件 I/O 序列化器迁移到 UTF-8 raw + +## 1 相关文档 +- [0827.md](0827.md) - 插件 I/O 协议统一为 UTF-8(上一步) +- [dddd.md](dddd.md) - 任务文档模板 + +## 2 任务相关的代码文件 +- `src/Data/Convert/Verbatim/verbatim.cpp` — 新增 `tree_to_utf8raw` / `utf8raw_to_tree` +- `src/Data/Convert/convert.hpp` — 声明新函数 +- `src/Scheme/L4/glue_convert.lua` — 暴露 `cpp-tree->utf8raw` / `cpp-utf8raw->tree` +- `TeXmacs/progs/convert/rewrite/init-rewrite.scm` — Scheme 包装 `texmacs->utf8raw` / `utf8raw->texmacs` +- `TeXmacs/progs/init-research.scm` — lazy-define 新包装函数 +- `TeXmacs/progs/utils/plugins/plugin-cmd.scm` — 默认序列化器重命名为 `utf8raw-serialize` 并改用 `texmacs->utf8raw` +- `TeXmacs/plugins/{python,julia,goldfish,autosave,tikz,quiver,gnuplot,maxima}/progs/init-*.scm` — 自定义 serializer 改用 `texmacs->utf8raw` +- `src/Data/Convert/Generic/input.cpp` — `verbatim_flush` / `utf8_flush` 改用 `utf8raw_to_tree` + `tree_utf8_to_herk` +- `src/System/Link/connection.cpp` — `connection_write(tree)` 入口加入 `tree_herk_to_utf8` +- `src/Edit/Interface/edit_complete.cpp` — `session_complete_command` 在调用 `utf8raw-serialize` 前加入 `tree_herk_to_utf8` +- `TeXmacs/progs/dynamic/session-edit.scm` — `input-done?` 路径在调用 `plugin-serialize` 前把 `pre` 从 Herk 转成 UTF-8 +- `TeXmacs/progs/dynamic/program-edit.scm` — 同上,针对 program session +- `TeXmacs/tests/0828.scm` — 新增 UTF-8 raw 往返、各插件 serializer UTF-8、输入/输出路径转换测试 + +## 3 如何测试 + +### 3.1 确定性测试(单元测试) +```bash +xmake b stem +xmake r 0801 +xmake r 0804 +xmake r 0827 +xmake r 0828 +``` + +### 3.2 非确定性测试(文档验证) + +## 4 What +- 新增无 cork/SourceCode 转换的 UTF-8 raw 序列化/反序列化函数。 +- 插件 I/O 主路径(`connection_write(tree)`)先把内部 Herk tree 转成 UTF-8 tree。 +- 所有插件 serializer 改用 `texmacs->utf8raw`,不再使用 `texmacs->code ... "SourceCode"` 或 `"utf-8"`。 +- 输入端 `verbatim_flush` / `utf8_flush` 改用 `utf8raw_to_tree`,再转回内部 Herk。 +- `tree_to_verbatim` / `verbatim_to_tree` 保留给文件格式转换等其他用途,plugin I/O 不再使用。 + +## 5 How +1. 在 `verbatim.cpp` 新增 `tree_to_utf8raw(tree)`,直接调用 `as_verbatim` 并处理 Windows 换行,跳过 `var_cork_to_utf8` / `var_cork_to_sourcecode`。 +2. 在 `verbatim.cpp` 新增 `utf8raw_to_tree(string)`,复用 `mac_to_unix` / `dos_to_unix` 与拆分逻辑,但不经过 `encode`(即不做 `utf8_to_cork`)。把原有拆分逻辑抽成静态 helper `verbatim_raw_to_tree` 供旧函数复用。两函数均不设 `wrap` 形参——`tree_to_verbatim` / `verbatim_to_tree` 保留了 `wrap` 是为了文件格式转换等需要折行的场景,而插件 I/O 一律传 `false`,故 raw 版本直接省去该形参,避免留下一处无调用者、无测试的折行分支。 +3. `convert.hpp` 声明两个新函数;`glue_convert.lua` 暴露给 Scheme。 +4. `init-rewrite.scm` 提供 `texmacs->utf8raw` / `utf8raw->texmacs` 包装;`init-research.scm` 加入 lazy-define。 +5. `plugin-cmd.scm` 的 `verbatim-serialize` 重命名为 `utf8raw-serialize`,`generic-serialize` 改用 `texmacs->utf8raw`。 +6. Python、Julia、Goldfish、Autosave、TikZ、Quiver、Gnuplot、Maxima 自定义 serializer 全部替换为 `texmacs->utf8raw (stree->tree u)`。 +7. `input.cpp` 的 `verbatim_flush` / `utf8_flush` 改为 `tree_utf8_to_herk (utf8raw_to_tree (buf))`。`texmacs_input_rep` 是插件连接的"输入端"——站在 TeXmacs 视角,插件进程经管道发回的数据要"输入"进来;`write(tree)` 把解析结果喂给**编辑器内部的文档树**(不是写文件,也不是发回插件)。编辑器内部所有字符串标签统一用 Herk 编码,所以 `write` 的实参必须是 Herk tree,而 `buf` 是插件发来的 UTF-8 字节,分两步翻译:`utf8raw_to_tree(buf)` 先切成 `(document ...)` 但标签仍是 UTF-8 原始字节(raw,不做编码转换),`tree_utf8_to_herk(...)` 再把每个 atomic 标签从 UTF-8 解码成 Herk。两步合起来等价于旧版 `verbatim_to_tree(buf, false, "utf-8")`(内部走 `utf8_to_cork`),但编码目标从 cork 升级为 Herk——cork 表之外的码点,旧的 `utf8_to_cork` 无映射就丢弃,新的 `utf8_to_herk` 用 `<#XXXX>` 保留,emoji、BMP 外字符因此不再丢失,这是相对 0827 的关键修正。此外 `utf8raw_to_tree` 引入了 `dos_to_unix` / `mac_to_unix` 换行归一化(旧 `verbatim_to_tree(buf,"utf-8")` 路径没有),插件输出里的 `\r\n` / 裸 `\r` 会被统一成 `\n`。写出端是对称的逆过程:第 8–10 条里编辑器内部的 Herk tree 先经 `tree_herk_to_utf8` 转成 UTF-8 tree,再交给 serializer 输出给插件。 +8. `connection.cpp` 的 `connection_write(tree)` 在 `tree_to_stree` 之前调用 `tree_herk_to_utf8`。 +9. `edit_complete.cpp` 的 `session_complete_command` 在调用 `utf8raw-serialize` 前加入 `tree_herk_to_utf8`,处理 tab completion 这条旁路。 +10. `session-edit.scm` 与 `program-edit.scm` 的 `input-done?` 路径在调用 `plugin-serialize` 前把 `pre` 从 Herk 转成 UTF-8,避免该查询路径发送 cork 文本给插件。**该转换不可省略:`plugin-serialize` 约定接收 UTF-8 stree**,而编辑器内部字符串为 Herk/cork,省略会让 Herk 的 `<#XXXX>` 字面原样漏给插件——插件收到的是 8 个 ASCII 字符 `<#4E2D>` 而非"中"。`0828.scm` 的 `test-input-done-herk-to-utf8` 用同一 Herk stree 对比了转换与不转换两种结果,断言不转换时 `<#XXXX>` 会漏出,以此证明该转换是必需的而非冗余。当前实现有 `stree->tree->stree` 往返开销,但 `input-done?` 输入通常很小,影响可忽略;若后续出现瓶颈,可改为直接 stree 级转换。 +11. 新增 `TeXmacs/tests/0828.scm`,覆盖 `texmacs->utf8raw` / `utf8raw->texmacs` 往返、各插件 serializer(含 Maxima 的 `;`/`$` 终止符与空输入特例)的 UTF-8 输出、输入端 Herk 转换(含 emoji 落到 `<#XXXX>` 的保留路径)、`<#XXXX>` 与普通文本混合的往返、主写出入口 Herk->UTF-8 转换、以及 `input-done?` 旁路 Herk stree 转 UTF-8 后再序列化(含不转换的反向对照)等场景。 + +## PS +- `TeXmacs/plugins/*/progs/data/*.scm` 中出现的 `(texmacs->verbatim x (acons "texmacs->verbatim:encoding" "SourceCode" '()))` 用于文件格式导入/导出(如 `.py`、`.cpp`、`.scm`),不属于插件 I/O 路径;SourceCode 编码用于文件持久化与往返,0828 不涉及这些路径,保持原样不改。 diff --git a/src/Data/Convert/Generic/input.cpp b/src/Data/Convert/Generic/input.cpp index 1f03d44d6a..20d62482f9 100644 --- a/src/Data/Convert/Generic/input.cpp +++ b/src/Data/Convert/Generic/input.cpp @@ -260,7 +260,7 @@ texmacs_input_rep::flush (bool force) { void texmacs_input_rep::verbatim_flush (bool force) { if (force || ends (buf, "\n")) { - if (!ignore_verb) write (verbatim_to_tree (buf, false, "utf-8")); + if (!ignore_verb) write (tree_utf8_to_herk (utf8raw_to_tree (buf))); else if (DEBUG_IO) debug_io << "ignore verbatim (aborted input)" << LF; buf= ""; } @@ -270,7 +270,7 @@ void texmacs_input_rep::utf8_flush (bool force) { if (force || ends (buf, "\n")) { if (!ignore_verb) { - write (verbatim_to_tree (buf, false, "utf-8")); + write (tree_utf8_to_herk (utf8raw_to_tree (buf))); } else if (DEBUG_IO) debug_io << "ignore verbatim (aborted input)" << LF; buf= ""; diff --git a/src/Data/Convert/Verbatim/verbatim.cpp b/src/Data/Convert/Verbatim/verbatim.cpp index 167f4de815..2da56782ad 100644 --- a/src/Data/Convert/Verbatim/verbatim.cpp +++ b/src/Data/Convert/Verbatim/verbatim.cpp @@ -271,6 +271,16 @@ tree_to_verbatim (tree t, bool wrap, string enc) { #endif } +string +tree_to_utf8raw (tree t) { + string buf= as_verbatim (t, false); +#ifdef OS_WIN + return unix_to_dos (buf); +#else + return buf; +#endif +} + /****************************************************************************** * Verbatim to TeXmacs ******************************************************************************/ @@ -302,9 +312,8 @@ encode (string s, string enc) { else return tm_encode (s); } -tree -verbatim_to_tree (string s, string enc) { - s= encode (s, enc); +static tree +verbatim_raw_to_tree (string s) { int i, j; int s_N= N (s); for (i= 0; i < s_N; i++) @@ -321,6 +330,12 @@ verbatim_to_tree (string s, string enc) { return un_special (s); } +tree +verbatim_to_tree (string s, string enc) { + s= encode (s, enc); + return verbatim_raw_to_tree (s); +} + string dos_to_unix (string s) { int i, n= N (s); @@ -344,6 +359,12 @@ mac_to_unix (string s) { return r; } +tree +utf8raw_to_tree (string s) { + s= mac_to_unix (dos_to_unix (s)); + return verbatim_raw_to_tree (s); +} + tree verbatim_to_tree (string s, bool wrap, string enc) { if (enc == "default") enc= "auto"; diff --git a/src/Data/Convert/convert.hpp b/src/Data/Convert/convert.hpp index 50942a7417..d45996dd66 100644 --- a/src/Data/Convert/convert.hpp +++ b/src/Data/Convert/convert.hpp @@ -61,4 +61,8 @@ tree verbatim_to_tree (string s, bool wrap= false, string enc= "default"); tree verbatim_document_to_tree (string s, bool w= false, string e= "default"); bool is_verbatim (tree t); +/*** UTF-8 raw (plugin I/O) ***/ +string tree_to_utf8raw (tree t); +tree utf8raw_to_tree (string s); + #endif // defined CONVERT_H diff --git a/src/Edit/Interface/edit_complete.cpp b/src/Edit/Interface/edit_complete.cpp index 4d0d0c71c2..5a419205c9 100644 --- a/src/Edit/Interface/edit_complete.cpp +++ b/src/Edit/Interface/edit_complete.cpp @@ -19,6 +19,7 @@ #include "observers.hpp" #include "preferences.hpp" #include "tree_observer.hpp" +#include "tree_traverse.hpp" #include @@ -355,7 +356,8 @@ edit_interface_rep::session_complete_command (tree tt) { (void) eval ("(use-modules (utils plugins plugin-cmd))"); string lan= get_env_string (PROG_LANGUAGE); string ses= get_env_string (PROG_SESSION); - string s = as_string (call ("verbatim-serialize", lan, tree_to_stree (t))); + tree u = tree_herk_to_utf8 (t); + string s = as_string (call ("utf8raw-serialize", lan, tree_to_stree (u))); s = s (0, N (s) - 1); int pos= search_forwards (cursor_symbol, s); diff --git a/src/Scheme/L4/glue_convert.lua b/src/Scheme/L4/glue_convert.lua index 3844ef8a27..4a374da662 100644 --- a/src/Scheme/L4/glue_convert.lua +++ b/src/Scheme/L4/glue_convert.lua @@ -130,6 +130,14 @@ function main() "string" } }, + { + scm_name = "cpp-tree->utf8raw", + cpp_name = "tree_to_utf8raw", + ret_type = "string", + arg_list = { + "tree" + } + }, { scm_name = "cpp-verbatim-snippet->texmacs", cpp_name = "verbatim_to_tree", @@ -140,6 +148,14 @@ function main() "string" } }, + { + scm_name = "cpp-utf8raw->tree", + cpp_name = "utf8raw_to_tree", + ret_type = "tree", + arg_list = { + "string" + } + }, { scm_name = "cpp-verbatim->texmacs", cpp_name = "verbatim_document_to_tree", diff --git a/src/System/Link/connection.cpp b/src/System/Link/connection.cpp index f93cd059ee..ea84b53de2 100644 --- a/src/System/Link/connection.cpp +++ b/src/System/Link/connection.cpp @@ -26,6 +26,7 @@ #include "socket_notifier.hpp" #include "tm_debug.hpp" #include "tree_helper.hpp" +#include "tree_traverse.hpp" using namespace moebius; @@ -258,7 +259,8 @@ connection_write (string name, string session, string s) { void connection_write (string name, string session, tree t) { // cout << "Write " << name << ", " << session << ", " << t << "\n"; - string s= as_string (call ("plugin-serialize", name, tree_to_stree (t))); + tree u= tree_herk_to_utf8 (t); + string s= as_string (call ("plugin-serialize", name, tree_to_stree (u))); connection_write (name, session, s); }