diff --git a/mcp_server/shared/code_tokenize.py b/mcp_server/shared/code_tokenize.py index 7845d281..5f944dcf 100644 --- a/mcp_server/shared/code_tokenize.py +++ b/mcp_server/shared/code_tokenize.py @@ -116,11 +116,20 @@ def expand_fts_query(query: str) -> str: precondition: ``query`` is a str. postcondition: returns an FTS5 MATCH expression that preserves the original - implicit-AND-across-words semantics (each source word becomes one required - group) while adding OR-alternatives for the sub-tokens of any camelCase / + AND-across-words semantics (each source word becomes one required group) + while adding OR-alternatives for the sub-tokens of any camelCase / snake_case word. Every term is quoted, so FTS5 operator keywords and punctuation in the input become harmless literals. Returns ``""`` when the query contains no indexable word (callers already guard the empty MATCH). + + Groups are joined with an explicit ``AND``, not whitespace: FTS5's grammar + rejects a parenthesized ``(a OR b)`` group immediately followed by a bare + phrase via implicit-AND (``(a OR b) "c"`` → "fts5: syntax error"), which + fired whenever a multi-token word (→ group) preceded a single-token word + (→ bare phrase) — e.g. an entity named ``cortex_viz/__main__.py`` crashed + get_causal_chain's entity-mention lookup on the SQLite backend. Explicit + ``AND`` is semantically identical (FTS5 ``a b`` ≡ ``a AND b``) and valid for + every group shape. """ groups: list[str] = [] for word in _WORD_RE.findall(query): @@ -133,4 +142,4 @@ def expand_fts_query(query: str) -> str: seen: set[str] = set() uniq = [a for a in alts if not (a in seen or seen.add(a))] groups.append("(" + " OR ".join(_fts_quote(a) for a in uniq) + ")") - return " ".join(groups) + return " AND ".join(groups) diff --git a/tests_py/shared/test_code_tokenize.py b/tests_py/shared/test_code_tokenize.py index 23976ab1..e72a36cc 100644 --- a/tests_py/shared/test_code_tokenize.py +++ b/tests_py/shared/test_code_tokenize.py @@ -9,6 +9,10 @@ from __future__ import annotations +import sqlite3 + +import pytest + from mcp_server.shared.code_tokenize import ( augment_content, expand_fts_query, @@ -78,8 +82,10 @@ def test_expand_query_quotes_and_expands(): def test_expand_query_preserves_and_across_words(): q = expand_fts_query("payment amount") - # two required groups, whitespace-joined (FTS5 implicit AND) - assert q == '"payment" "amount"' + # two required groups, joined with an explicit AND (FTS5 grammar rejects + # implicit-AND when either side is a parenthesized group — see regression + # test below). + assert q == '"payment" AND "amount"' def test_expand_query_fts5_keyword_is_literal(): @@ -89,3 +95,63 @@ def test_expand_query_fts5_keyword_is_literal(): def test_expand_query_empty(): assert expand_fts_query("!!!") == "" + + +# --- Regression: FTS5 grammar acceptance (get_causal_chain crash) ----------- +# The bug: expand_fts_query joined its groups with whitespace, so a query +# mixing a multi-token word (→ "(a OR b)" group) with a single-token word +# (→ bare "c" phrase) produced "(a OR b) \"c\"", which FTS5 rejects with +# 'fts5: syntax error near ...'. An entity named cortex_viz/__main__.py hit +# exactly this on the SQLite backend and crashed get_causal_chain. + + +def _fts5_available() -> bool: + try: + con = sqlite3.connect(":memory:") + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.close() + return True + except sqlite3.OperationalError: + return False + + +@pytest.mark.skipif(not _fts5_available(), reason="sqlite3 built without FTS5") +@pytest.mark.parametrize( + "query", + [ + "cortex_viz/__main__.py", # group then bare phrase — the original crash + "__main__ cortex_viz", # bare phrase then group — the mirror case + "normalizePaymentAmount rounds the total", # group amid plain words + "utf8 payment snake_case_id", # multiple groups and phrases interleaved + ], +) +def test_expanded_query_is_accepted_by_real_fts5(query: str) -> None: + match = expand_fts_query(query) + assert match # each query has at least one indexable word + con = sqlite3.connect(":memory:") + try: + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex viz main payment",)) + # The assertion is that MATCH parses at all — no OperationalError. + con.execute("SELECT rowid FROM t WHERE t MATCH ?", (match,)).fetchall() + finally: + con.close() + + +@pytest.mark.skipif(not _fts5_available(), reason="sqlite3 built without FTS5") +def test_group_and_phrase_are_and_joined() -> None: + # A group followed by a bare phrase must be joined so both are REQUIRED: + # the row matches only when it contains a sub-token of the group AND the + # phrase — proving the join is a conjunction, not a dropped/broken clause. + match = expand_fts_query("cortex_viz rounds") + con = sqlite3.connect(":memory:") + try: + con.execute("CREATE VIRTUAL TABLE t USING fts5(x)") + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex rounds the total",)) + con.execute("INSERT INTO t(x) VALUES (?)", ("cortex only",)) # no 'rounds' + rows = con.execute( + "SELECT x FROM t WHERE t MATCH ? ORDER BY rowid", (match,) + ).fetchall() + assert rows == [("cortex rounds the total",)] + finally: + con.close()