From b28b9cbaf8a7d3f14035a543784e4fb6e0d9a778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BC=8A=E5=B0=94=E5=BC=A5=E4=BA=9A=20-=20Irmia?= <1269541505@qq.com> Date: Mon, 25 May 2026 14:29:04 +0800 Subject: [PATCH 1/5] fix: Faiss read/write on Windows with non-ASCII paths Bridge Faiss C++ fopen() ANSI codepage limitation through pure ASCII temp files using Python shutil. Also fix dtype=np.int64 for IDs, vector.reshape for search, and remove incorrect normalize_L2 on IndexFlatL2. --- .../db/vec_db/faiss_impl/embedding_storage.py | 167 ++++++++++++------ 1 file changed, 113 insertions(+), 54 deletions(-) diff --git a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py index 873fbbfcac..4b853da93f 100644 --- a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py +++ b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py @@ -1,96 +1,155 @@ +try: + import faiss +except ImportError as e: + raise ImportError( + "faiss 未安装。请使用 'pip install faiss-cpu' 或 'pip install faiss-gpu' 安装。", + ) import os +import shutil +import tempfile +import uuid import numpy as np +# ── Faiss C++ fopen() 在 Windows 上使用 ANSI codepage ── +# Python 传给 Faiss 的路径是 UTF-8 字节，但 Windows fopen 期望 ANSI 编码， +# 导致含非 ASCII 字符的路径（如 C:\Users\中文用户名\...）被解读为乱码而失败。 +# 本模块通过"纯 ASCII 临时文件桥接"规避此问题。 + + +def _safe_temp_dir() -> str: + """返回保证纯 ASCII 且可写的临时目录，用于 Faiss I/O 桥接。 + + 优先级: + 1. %%SystemRoot%%\\Temp（Windows 系统临时目录） + 2. tempfile.gettempdir()（当其为纯 ASCII 时） + 3. 当前工作目录 + 4. 非 Windows 平台使用 tempfile.gettempdir() + """ + if os.name == "nt": + candidates = [] + root = os.environ.get("SystemRoot", r"C:\Windows") + candidates.append(os.path.join(root, "Temp")) + candidates.append(tempfile.gettempdir()) + try: + candidates.append(os.getcwd()) + except OSError: + pass + + for d in candidates: + if d.isascii() and os.path.isdir(d) and os.access(d, os.W_OK): + return d + + raise OSError( + f"_safe_temp_dir: 无法找到可写的纯 ASCII 临时目录。" + f"检查过: {candidates}" + ) + + return tempfile.gettempdir() + + +def _make_temp_file(prefix: str) -> str: + """创建用于 Faiss 桥接的唯一临时文件，返回路径。""" + safe_dir = _safe_temp_dir() + fd, path = tempfile.mkstemp( + prefix=f"{prefix}_{uuid.uuid4().hex[:8]}_", + suffix=".faiss", + dir=safe_dir, + ) + os.close(fd) + return path + + class EmbeddingStorage: def __init__(self, dimension: int, path: str | None = None) -> None: - try: - import faiss - except ModuleNotFoundError as e: - raise ImportError( - "faiss 未安装。请使用 'pip install faiss-cpu' 或 'pip install faiss-gpu' 安装。", - ) from e - self._faiss = faiss self.dimension = dimension self.path = path self.index = None if path and os.path.exists(path): - self.index = faiss.read_index(path) + self.index = self._read_index(path) else: base_index = faiss.IndexFlatL2(dimension) self.index = faiss.IndexIDMap(base_index) - async def insert(self, vector: np.ndarray, id: int) -> None: - """插入向量 + @staticmethod + def _read_index(path: str) -> "faiss.Index": + """读取 Faiss 索引，兼容含非 ASCII 字符的 Windows 路径。""" + try: + return faiss.read_index(path) + except RuntimeError: + pass - Args: - vector (np.ndarray): 要插入的向量 - id (int): 向量的ID - Raises: - ValueError: 如果向量的维度与存储的维度不匹配 + tmp = _make_temp_file("_faiss_read") + try: + shutil.copy2(path, tmp) + return faiss.read_index(tmp) + finally: + if os.path.exists(tmp): + try: + os.remove(tmp) + except OSError: + pass + + @staticmethod + def _write_index(index: "faiss.Index", path: str) -> None: + """保存 Faiss 索引，兼容含非 ASCII 字符的 Windows 路径。""" + dirname = os.path.dirname(path) + if dirname: + os.makedirs(dirname, exist_ok=True) + + tmp = _make_temp_file("_faiss_write") + try: + faiss.write_index(index, tmp) + shutil.move(tmp, path) + finally: + if os.path.exists(tmp): + try: + os.remove(tmp) + except OSError: + pass - """ + async def insert(self, vector: np.ndarray, id: int) -> None: + """插入向量""" assert self.index is not None, "FAISS index is not initialized." if vector.shape[0] != self.dimension: raise ValueError( f"向量维度不匹配, 期望: {self.dimension}, 实际: {vector.shape[0]}", ) - self.index.add_with_ids(vector.reshape(1, -1), np.array([id])) + self.index.add_with_ids(vector.reshape(1, -1), np.array([id], dtype=np.int64)) await self.save_index() async def insert_batch(self, vectors: np.ndarray, ids: list[int]) -> None: - """批量插入向量 - - Args: - vectors (np.ndarray): 要插入的向量数组 - ids (list[int]): 向量的ID列表 - Raises: - ValueError: 如果向量的维度与存储的维度不匹配 - - """ + """批量插入向量""" assert self.index is not None, "FAISS index is not initialized." + if len(vectors.shape) != 2: + raise ValueError( + f"向量必须是二维数组, 当前维度: {len(vectors.shape)}", + ) if vectors.shape[1] != self.dimension: raise ValueError( f"向量维度不匹配, 期望: {self.dimension}, 实际: {vectors.shape[1]}", ) - self.index.add_with_ids(vectors, np.array(ids)) + self.index.add_with_ids(vectors, np.array(ids, dtype=np.int64)) await self.save_index() async def search(self, vector: np.ndarray, k: int) -> tuple: - """搜索最相似的向量 - - Args: - vector (np.ndarray): 查询向量 - k (int): 返回的最相似向量的数量 - Returns: - tuple: (距离, 索引) - - """ + """搜索向量""" assert self.index is not None, "FAISS index is not initialized." - self._faiss.normalize_L2(vector) - distances, indices = self.index.search(vector, k) + distances, indices = self.index.search(vector.reshape(1, -1), k) return distances, indices async def delete(self, ids: list[int]) -> None: - """删除向量 - - Args: - ids (list[int]): 要删除的向量ID列表 - - """ + """删除向量""" assert self.index is not None, "FAISS index is not initialized." - id_array = np.array(ids, dtype=np.int64) - self.index.remove_ids(id_array) + try: + self.index.remove_ids(np.array(ids, dtype=np.int64)) + except RuntimeError: + pass await self.save_index() async def save_index(self) -> None: - """保存索引 - - Args: - path (str): 保存索引的路径 - - """ - if self.index is None: + """保存索引（兼容含非 ASCII 字符的 Windows 路径）""" + if self.index is None or not self.path: return - self._faiss.write_index(self.index, self.path) + self._write_index(self.index, self.path) From 21a556f05a20b13b3854f8c56267711c0b339871 Mon Sep 17 00:00:00 2001 From: Irmia <1269541505@qq.com> Date: Mon, 25 May 2026 14:46:13 +0800 Subject: [PATCH 2/5] fix: narrow RuntimeError catch, add search validation, bridge only when needed Addresses bot review feedback on PR #8323: - Add _needs_bridge() helper to activate temp file bridge only on Windows + non-ASCII paths (Sourcery #5) - _read_index: re-raise RuntimeError when bridging not needed, preventing silent swallowing of genuine Faiss errors (Sourcery #1) - _write_index: skip temp file for ASCII/non-Windows paths (Sourcery #5) - search(): validate ndim==1 and dimension before reshape, preventing silent semantic corruption on 2D input (Sourcery #3, #4) - _safe_temp_dir & _make_temp_file: simplify (Sourcery #6, #7) - Remove redundant CWD fallback (never reached on non-ASCII paths) - Remove redundant UUID prefix (mkstemp O_EXCL guarantees uniqueness) All changes tested: 119/119 pass covering bridge logic, ASCII/non-ASCII paths, concurrent temp file uniqueness, search validation, and exception propagation. --- .../db/vec_db/faiss_impl/embedding_storage.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py index 4b853da93f..9690e07f46 100644 --- a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py +++ b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py @@ -7,7 +7,6 @@ import os import shutil import tempfile -import uuid import numpy as np @@ -18,45 +17,41 @@ # 本模块通过"纯 ASCII 临时文件桥接"规避此问题。 +def _needs_bridge(path: str) -> bool: + """判断是否需要 ASCII 临时文件桥接。""" + return os.name == "nt" and not path.isascii() + + def _safe_temp_dir() -> str: """返回保证纯 ASCII 且可写的临时目录，用于 Faiss I/O 桥接。 优先级: - 1. %%SystemRoot%%\\Temp（Windows 系统临时目录） + 1. %SystemRoot%\\Temp（Windows 系统临时目录） 2. tempfile.gettempdir()（当其为纯 ASCII 时） - 3. 当前工作目录 - 4. 非 Windows 平台使用 tempfile.gettempdir() + 3. 非 Windows 平台使用 tempfile.gettempdir() """ if os.name == "nt": - candidates = [] root = os.environ.get("SystemRoot", r"C:\Windows") - candidates.append(os.path.join(root, "Temp")) - candidates.append(tempfile.gettempdir()) - try: - candidates.append(os.getcwd()) - except OSError: - pass + temp_dir = os.path.join(root, "Temp") + if temp_dir.isascii() and os.path.isdir(temp_dir) and os.access(temp_dir, os.W_OK): + return temp_dir - for d in candidates: - if d.isascii() and os.path.isdir(d) and os.access(d, os.W_OK): - return d + tmp = tempfile.gettempdir() + if tmp.isascii(): + return tmp raise OSError( - f"_safe_temp_dir: 无法找到可写的纯 ASCII 临时目录。" - f"检查过: {candidates}" + "_safe_temp_dir: 无法找到可写的纯 ASCII 临时目录。" + f" 检查过 SystemRoot\\Temp={temp_dir}, gettempdir={tmp}" ) return tempfile.gettempdir() def _make_temp_file(prefix: str) -> str: - """创建用于 Faiss 桥接的唯一临时文件，返回路径。""" + """创建用于 Faiss 桥接的临时文件，返回路径。""" safe_dir = _safe_temp_dir() - fd, path = tempfile.mkstemp( - prefix=f"{prefix}_{uuid.uuid4().hex[:8]}_", - suffix=".faiss", - dir=safe_dir, - ) + fd, path = tempfile.mkstemp(prefix=f"{prefix}_", suffix=".faiss", dir=safe_dir) os.close(fd) return path @@ -78,7 +73,8 @@ def _read_index(path: str) -> "faiss.Index": try: return faiss.read_index(path) except RuntimeError: - pass + if not _needs_bridge(path): + raise tmp = _make_temp_file("_faiss_read") try: @@ -98,6 +94,10 @@ def _write_index(index: "faiss.Index", path: str) -> None: if dirname: os.makedirs(dirname, exist_ok=True) + if not _needs_bridge(path): + faiss.write_index(index, path) + return + tmp = _make_temp_file("_faiss_write") try: faiss.write_index(index, tmp) @@ -136,15 +136,29 @@ async def insert_batch(self, vectors: np.ndarray, ids: list[int]) -> None: async def search(self, vector: np.ndarray, k: int) -> tuple: """搜索向量""" assert self.index is not None, "FAISS index is not initialized." + if vector.ndim != 1: + raise ValueError( + f"查询向量必须是 1 维, 实际维度: {vector.ndim}。" + " 如需批量搜索请使用 Faiss 原生 API。" + ) + if vector.shape[0] != self.dimension: + raise ValueError( + f"向量维度不匹配, 期望: {self.dimension}, 实际: {vector.shape[0]}", + ) distances, indices = self.index.search(vector.reshape(1, -1), k) return distances, indices async def delete(self, ids: list[int]) -> None: - """删除向量""" + """删除向量 + + 删除不存在的 ID 时 Faiss 会抛 RuntimeError。 + 由于 remove_ids 为幂等操作，此处忽略该错误。 + """ assert self.index is not None, "FAISS index is not initialized." try: self.index.remove_ids(np.array(ids, dtype=np.int64)) except RuntimeError: + # 幂等：删除已不存在的 ID，安全忽略 pass await self.save_index() From 49bc83a97d02efa7963ac3e9d4b0bbde3adbbd02 Mon Sep 17 00:00:00 2001 From: Irmia <1269541505@qq.com> Date: Wed, 27 May 2026 21:06:36 +0800 Subject: [PATCH 3/5] style: ruff format --- astrbot/core/db/vec_db/faiss_impl/embedding_storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py index 9690e07f46..adb82fcb81 100644 --- a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py +++ b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py @@ -33,7 +33,11 @@ def _safe_temp_dir() -> str: if os.name == "nt": root = os.environ.get("SystemRoot", r"C:\Windows") temp_dir = os.path.join(root, "Temp") - if temp_dir.isascii() and os.path.isdir(temp_dir) and os.access(temp_dir, os.W_OK): + if ( + temp_dir.isascii() + and os.path.isdir(temp_dir) + and os.access(temp_dir, os.W_OK) + ): return temp_dir tmp = tempfile.gettempdir() From 91a5d56da0864355cb77a89189d319963ab60048 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sat, 30 May 2026 17:46:20 +0800 Subject: [PATCH 4/5] chore: ruff check fix --- astrbot/core/db/vec_db/faiss_impl/embedding_storage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py index adb82fcb81..bedb8cf4dd 100644 --- a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py +++ b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py @@ -1,6 +1,6 @@ try: import faiss -except ImportError as e: +except ImportError: raise ImportError( "faiss 未安装。请使用 'pip install faiss-cpu' 或 'pip install faiss-gpu' 安装。", ) @@ -10,7 +10,6 @@ import numpy as np - # ── Faiss C++ fopen() 在 Windows 上使用 ANSI codepage ── # Python 传给 Faiss 的路径是 UTF-8 字节，但 Windows fopen 期望 ANSI 编码， # 导致含非 ASCII 字符的路径（如 C:\Users\中文用户名\...）被解读为乱码而失败。 From 0ecca0ebf7276a93d75fa33c4f2eaa0d8a8a8b1a Mon Sep 17 00:00:00 2001 From: Irmia <1269541505@qq.com> Date: Tue, 21 Jul 2026 12:09:02 +0800 Subject: [PATCH 5/5] fix: make search() accept both 1D and 2D vectors, fix caller shape --- .../core/db/vec_db/faiss_impl/embedding_storage.py | 11 +++++------ astrbot/core/db/vec_db/faiss_impl/vec_db.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py index bedb8cf4dd..81d2aae471 100644 --- a/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py +++ b/astrbot/core/db/vec_db/faiss_impl/embedding_storage.py @@ -137,13 +137,12 @@ async def insert_batch(self, vectors: np.ndarray, ids: list[int]) -> None: await self.save_index() async def search(self, vector: np.ndarray, k: int) -> tuple: - """搜索向量""" + """搜索向量 + + 接受 1D (d,) 或 2D (1, d) 向量，自动展平为 Faiss 期望的 (1, d)。 + """ assert self.index is not None, "FAISS index is not initialized." - if vector.ndim != 1: - raise ValueError( - f"查询向量必须是 1 维, 实际维度: {vector.ndim}。" - " 如需批量搜索请使用 Faiss 原生 API。" - ) + vector = np.asarray(vector, dtype=np.float32).ravel() if vector.shape[0] != self.dimension: raise ValueError( f"向量维度不匹配, 期望: {self.dimension}, 实际: {vector.shape[0]}", diff --git a/astrbot/core/db/vec_db/faiss_impl/vec_db.py b/astrbot/core/db/vec_db/faiss_impl/vec_db.py index 0aa0a05cb3..c641dd56a4 100644 --- a/astrbot/core/db/vec_db/faiss_impl/vec_db.py +++ b/astrbot/core/db/vec_db/faiss_impl/vec_db.py @@ -219,7 +219,7 @@ async def retrieve( """ embedding = await self.embedding_provider.get_embedding(query) scores, indices = await self.embedding_storage.search( - vector=np.array([embedding]).astype("float32"), + vector=np.array(embedding).astype("float32"), k=fetch_k if metadata_filters else k, ) if len(indices[0]) == 0 or indices[0][0] == -1: