From 62ef41506f825d756f144cebec60cc4b853c4cb4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 17:04:31 +0000 Subject: [PATCH] fix(fts): AND-join expand_fts_query groups so mixed group+phrase parses expand_fts_query joined its per-word groups with whitespace. FTS5's grammar accepts implicit-AND between two bare phrases (a b) but rejects it when either operand is a parenthesized group: (a OR b) "c" raises "fts5: syntax error". This fired whenever a multi-token identifier (which becomes an OR group) sat next to a single-token word (a bare phrase) in the same query. A knowledge-graph entity named cortex_viz/__main__.py hit it exactly -- get_causal_chain's entity-mention lookup crashed on the SQLite backend with 'fts5: syntax error near ""__main__""'. Fix: join groups with an explicit AND. FTS5 treats "a b" and "a AND b" as identical, but the explicit form is grammatically valid for every group shape, so the crash is gone with no change in match semantics. Regression tests run expand_fts_query output against a real in-memory FTS5 table (parametrized over the group-then-phrase, phrase-then-group, and interleaved cases) asserting no OperationalError, plus one test proving the join is a conjunction (group AND phrase both required). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_017fnomaXS71hgxMw2zr7HGR --- mcp_server/shared/code_tokenize.py | 15 ++++-- tests_py/shared/test_code_tokenize.py | 70 ++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/mcp_server/shared/code_tokenize.py b/mcp_server/shared/code_tokenize.py index 7845d281..5f944dcf 100644 --- a/mcp_server/shared/code_tokenize.py +++ b/mcp_server/shared/code_tokenize.py @@ -116,11 +116,20 @@ def expand_fts_query(query: str) -> str: precondition: ``query`` is a str. postcondition: returns an FTS5 MATCH expression that preserves the original - implicit-AND-across-words semantics (each source word becomes one required - group) while adding OR-alternatives for the sub-tokens of any camelCase / + AND-across-words semantics (each source word becomes one required group) + while adding OR-alternatives for the sub-tokens of any camelCase / snake_case word. Every term is quoted, so FTS5 operator keywords and punctuation in the input become harmless literals. Returns ``""`` when the query contains no indexable word (callers already guard the empty MATCH). + + Groups are joined with an explicit ``AND``, not whitespace: FTS5's grammar + rejects a parenthesized ``(a OR b)`` group immediately followed by a bare + phrase via implicit-AND (``(a OR b) "c"`` → "fts5: syntax error"), which + fired whenever a multi-token word (→ group) preceded a single-token word + (→ bare phrase) — e.g. an entity named ``cortex_viz/__main__.py`` crashed + get_causal_chain's entity-mention lookup on the SQLite backend. Explicit + ``AND`` is semantically identical (FTS5 ``a b`` ≡ ``a AND b``) and valid for + every group shape. """ groups: list[str] = [] for word in _WORD_RE.findall(query): @@ -133,4 +142,4 @@ def expand_fts_query(query: str) -> str: seen: set[str] = set() uniq = [a for a in alts if not (a in seen or seen.add(a))] groups.append("(" + " OR ".join(_fts_quote(a) for a in uniq) + ")") - return " ".join(groups) + return " AND ".join(groups) diff --git a/tests_py/shared/test_code_tokenize.py b/tests_py/shared/test_code_tokenize.py index 23976ab1..e72a36cc 100644 --- a/tests_py/shared/test_code_tokenize.py +++ b/tests_py/shared/test_code_tokenize.py @@ -9,6 +9,10 @@ from __future__ import annotations +import sqlite3 + +import pytest + from mcp_server.shared.code_tokenize import ( augment_content, expand_fts_query, @@ -78,8 +82,10 @@ def test_expand_query_quotes_and_expands(): def test_expand_query_preserves_and_across_words(): q = expand_fts_query("payment amount") - # two required groups, whitespace-joined (FTS5 implicit AND) - assert q == '"payment" "amount"' + # two required groups, joined with an explicit AND (FTS5 grammar rejects + # implicit-AND when either side is a parenthesized group — see regression + # test below). + assert q == '"payment" AND "amount"' def test_expand_query_fts5_keyword_is_literal(): @@ -89,3 +95,63 @@ def test_expand_query_fts5_keyword_is_literal(): def test_expand_query_empty(): assert expand_fts_query("!!!") == "" + + +# --- Regression: FTS5 grammar acceptance (get_causal_chain crash) ----------- +# The bug: expand_fts_query joined its groups with whitespace, so a query +# mixing a multi-token word (→ "(a OR b)" group) with a single-token word +# (→ bare "c" phrase) produced "(a OR b) \"c\"", which FTS5 rejects with +# 'fts5: syntax error near ...'. An entity named cortex_viz/__main__.py hit +# exactly this on the SQLite backend and crashed get_causal_chain. + + +def _fts5_available() -> bool: + try: + con = sqlite3.connect(":memory:") + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.close() + return True + except sqlite3.OperationalError: + return False + + +@pytest.mark.skipif(not _fts5_available(), reason="sqlite3 built without FTS5") +@pytest.mark.parametrize( + "query", + [ + "cortex_viz/__main__.py", # group then bare phrase — the original crash + "__main__ cortex_viz", # bare phrase then group — the mirror case + "normalizePaymentAmount rounds the total", # group amid plain words + "utf8 payment snake_case_id", # multiple groups and phrases interleaved + ], +) +def test_expanded_query_is_accepted_by_real_fts5(query: str) -> None: + match = expand_fts_query(query) + assert match # each query has at least one indexable word + con = sqlite3.connect(":memory:") + try: + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex viz main payment",)) + # The assertion is that MATCH parses at all — no OperationalError. + con.execute("SELECT rowid FROM t WHERE t MATCH ?", (match,)).fetchall() + finally: + con.close() + + +@pytest.mark.skipif(not _fts5_available(), reason="sqlite3 built without FTS5") +def test_group_and_phrase_are_and_joined() -> None: + # A group followed by a bare phrase must be joined so both are REQUIRED: + # the row matches only when it contains a sub-token of the group AND the + # phrase — proving the join is a conjunction, not a dropped/broken clause. + match = expand_fts_query("cortex_viz rounds") + con = sqlite3.connect(":memory:") + try: + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex rounds the total",)) + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex only",)) # no 'rounds' + rows = con.execute( + "SELECT x FROM t WHERE t MATCH ? ORDER BY rowid", (match,) + ).fetchall() + assert rows == [("cortex rounds the total",)] + finally: + con.close()