feat(search): index prose content for BM25 full-text search

internal/cbm/extract_defs.c

@@ -3861,7 +3861,132 @@ static const char *qn_safe_segment(CBMArena *a, const char *name) {
     return out;
 }
 
-static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
+/* UTF-8 sequence classification: a lead byte's high bits name the sequence
+ * length, and every continuation byte matches 10xxxxxx. */
+enum {
+    UTF8_CONT_MASK = 0xC0,  /* isolate the two high bits ... */
+    UTF8_CONT_MARK = 0x80,  /* ... which are 10 on a continuation byte */
+    UTF8_LEAD2_MASK = 0xE0, /* 110xxxxx → 2-byte sequence */
+    UTF8_LEAD2_MARK = 0xC0,
+    UTF8_LEAD3_MASK = 0xF0, /* 1110xxxx → 3-byte sequence */
+    UTF8_LEAD3_MARK = 0xE0,
+    UTF8_LEAD4_MASK = 0xF8, /* 11110xxx → 4-byte sequence */
+    UTF8_LEAD4_MARK = 0xF0,
+    UTF8_LEN_1 = 1,
+    UTF8_LEN_2 = 2,
+    UTF8_LEN_3 = 3,
+    UTF8_LEN_4 = 4,
+};
+
+/* Bytes the sequence starting with `lead` occupies (1 for ASCII or a byte that
+ * is not a valid lead). */
+static size_t utf8_sequence_len(unsigned char lead) {
+    if ((lead & UTF8_LEAD2_MASK) == UTF8_LEAD2_MARK) {
+        return UTF8_LEN_2;
+    }
+    if ((lead & UTF8_LEAD3_MASK) == UTF8_LEAD3_MARK) {
+        return UTF8_LEN_3;
+    }
+    if ((lead & UTF8_LEAD4_MASK) == UTF8_LEAD4_MARK) {
+        return UTF8_LEN_4;
+    }
+    return UTF8_LEN_1;
+}
+
+/* Drop a trailing PARTIAL UTF-8 sequence left behind by a byte-length cut, so
+ * a capped prose value never ends mid-codepoint (#1017's rule, applied to the
+ * prose bodies below rather than to comments). */
+static void utf8_trim_partial_tail(char *text) {
+    size_t n = strlen(text);
+    if (n == 0) {
+        return;
+    }
+    size_t i = n;
+    while (i > 0 && ((unsigned char)text[i - UTF8_LEN_1] & UTF8_CONT_MASK) == UTF8_CONT_MARK) {
+        i--;
+    }
+    if (i == 0) {
+        text[0] = '\0'; /* continuation bytes only — not decodable */
+        return;
+    }
+    size_t need = utf8_sequence_len((unsigned char)text[i - UTF8_LEN_1]);
+    if (i - UTF8_LEN_1 + need > n) {
+        text[i - UTF8_LEN_1] = '\0';
+    }
+}
+
+/* Collapse `len` bytes of raw prose into a single-spaced value capped at
+ * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which
+ * leaves room inside the 2 KB properties buffer that carries it).
+ *
+ * The output buffer is FIXED at that cap, so a section body of any size costs a
+ * bounded copy rather than a copy of the whole section. Returns NULL when
+ * nothing but whitespace was there. */
+static char *collapse_prose(CBMArena *a, const char *src, size_t len) {
+    if (!src || len == 0) {
+        return NULL;
+    }
+    char *out = (char *)cbm_arena_alloc(a, MAX_COMMENT_LEN + NULL_TERM);
+    if (!out) {
+        return NULL;
+    }
+    size_t w = 0;
+    bool in_ws = true; /* start true so leading whitespace is swallowed */
+    for (size_t i = 0; i < len && w < MAX_COMMENT_LEN; i++) {
+        unsigned char c = (unsigned char)src[i];
+        if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
+            in_ws = true;
+            continue;
+        }
+        if (in_ws && w > 0) {
+            out[w++] = ' ';
+            if (w >= MAX_COMMENT_LEN) {
+                break;
+            }
+        }
+        in_ws = false;
+        out[w++] = (char)c;
+    }
+    out[w] = '\0';
+    utf8_trim_partial_tail(out);
+    return out[0] ? out : NULL;
+}
+
+static bool is_markdown_heading_kind(const char *kind) {
+    return strcmp(kind, "atx_heading") == 0 || strcmp(kind, "setext_heading") == 0;
+}
+
+/* #518: a Markdown heading node is only the title line — the section's prose
+ * lives in the blocks that FOLLOW it. Collect that prose as the Section's
+ * docstring so the node carries the text a reader would search for; nodes_fts
+ * indexes it from there (`body`), which is the whole point of the issue.
+ *
+ * tree-sitter-markdown wraps a heading and its content in a `section` node,
+ * with nested subsections as further `section` children that own their own
+ * text. Stopping at either a `section` or another heading therefore gives each
+ * Section exactly its OWN body under both that shape and a flat one. The bytes
+ * between are contiguous in the source, so one slice beats concatenation. */
+static const char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) {
+    uint32_t start = ts_node_end_byte(heading);
+    uint32_t end = start;
+    for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib);
+         sib = ts_node_next_sibling(sib)) {
+        const char *sk = ts_node_type(sib);
+        if (strcmp(sk, "section") == 0 || is_markdown_heading_kind(sk)) {
+            break;
+        }
+        end = ts_node_end_byte(sib);
+    }
+    if (end <= start) {
+        return NULL;
+    }
+    return collapse_prose(a, source + start, (size_t)(end - start));
+}
+
+/* Push a config-language definition that carries prose. push_simple_class_def
+ * delegates here with no docstring. */
+static void push_simple_class_def_doc(CBMExtractCtx *ctx, TSNode node, char *name,
+                                      const char *label, const char *docstring) {
     CBMArena *a = ctx->arena;
     CBMDefinition def;
     memset(&def, 0, sizeof(def));
@@ -3872,9 +3997,14 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c
     def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;
     def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;
     def.is_exported = true;
+    def.docstring = docstring;
     cbm_defs_push(&ctx->result->defs, a, def);
 }
 
+static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
+    push_simple_class_def_doc(ctx, node, name, label, NULL);
+}
+
 // Find TOML table key name from children.
 static char *find_toml_key_name(CBMArena *a, TSNode node, const char *source) {
     uint32_t nc = ts_node_child_count(node);
@@ -4033,6 +4163,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
     CBMArena *a = ctx->arena;
     char *name = NULL;
     const char *label = "Class";
+    const char *docstring = NULL;
 
     if (ctx->language == CBM_LANG_TOML &&
         (strcmp(kind, "table") == 0 || strcmp(kind, "table_array_element") == 0)) {
@@ -4048,14 +4179,16 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
         // label rather than degrade it to match a test. The markdown repro asserts
         // "Class"; that assertion is the inaccurate side and is flagged for review.
         label = "Section";
+        // #518: index what the section SAYS, not just its title.
+        docstring = extract_markdown_section_body(a, node, ctx->source);
     } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) {
         name = find_hcl_block_name(a, node, ctx->source);
     } else {
         return false;
     }
 
     if (name && name[0]) {
-        push_simple_class_def(ctx, node, name, label);
+        push_simple_class_def_doc(ctx, node, name, label, docstring);
     }
     return true;
 }
@@ -6175,12 +6308,9 @@ static bool is_helm_values_file(const char *rel) {
     return strcmp(b, "values.yaml") == 0 || strcmp(b, "values.yml") == 0;
 }
 
-// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
-// Helm values.yaml so each chart's tunables surface as a handful of structured
-// Variables instead of one node per nested leaf (#338).
-static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
-    CBMArena *a = ctx->arena;
-    // Descend stream -> document -> block_node down to the first block_mapping.
+// Descend stream -> document -> block_node to a YAML document's top-level
+// block_mapping. Returns a null node when the document has none.
+static TSNode find_yaml_toplevel_mapping(TSNode root) {
     TSNode bm = {0};
     TSNode cur = root;
     for (int depth = 0; depth < 6 && ts_node_is_null(bm); depth++) {
@@ -6204,6 +6334,126 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
         }
         cur = next;
     }
+    return bm;
+}
+
+// Descend to a JSON document's top-level object. Returns a null node if absent.
+static TSNode find_json_toplevel_object(TSNode root) {
+    if (strcmp(ts_node_type(root), "object") == 0) {
+        return root;
+    }
+    TSNode obj = cbm_find_child_by_kind(root, "object");
+    if (!ts_node_is_null(obj)) {
+        return obj;
+    }
+    uint32_t n = ts_node_named_child_count(root);
+    for (uint32_t i = 0; i < n; i++) {
+        TSNode inner = cbm_find_child_by_kind(ts_node_named_child(root, i), "object");
+        if (!ts_node_is_null(inner)) {
+            return inner;
+        }
+    }
+    TSNode null_node = {0};
+    return null_node;
+}
+
+/* #519: a config file's own prose sits in a top-level `description` (or one of
+ * its usual synonyms) — META.yaml, action.yml, an OpenAPI document,
+ * package.json. Nothing indexed it: the VALUE is not a definition, so no node
+ * carried it and BM25 could not see it at all. Checked in priority order; the
+ * first key present wins. */
+static const char *const config_desc_keys[] = {"description", "summary", "purpose", NULL};
+
+// Strip one layer of matching surrounding quotes. Operates on arena text.
+static char *strip_surrounding_quotes(char *t) {
+    if (!t) {
+        return t;
+    }
+    size_t n = strlen(t);
+    if (n >= PAIR_CHARS && (t[0] == '"' || t[0] == '\'') && t[n - SKIP_CHAR] == t[0]) {
+        t[n - SKIP_CHAR] = '\0';
+        return t + SKIP_CHAR;
+    }
+    return t;
+}
+
+// Normalise a config scalar into an indexable value: drop a YAML block-scalar
+// header (`|`/`>` plus its chomping and indent modifiers), collapse whitespace
+// to the shared MAX_COMMENT_LEN cap, then unquote.
+static const char *config_scalar_value(CBMArena *a, const char *raw) {
+    if (!raw) {
+        return NULL;
+    }
+    while (*raw == ' ' || *raw == '\t' || *raw == '\n' || *raw == '\r') {
+        raw++;
+    }
+    if (*raw == '|' || *raw == '>') {
+        while (*raw && *raw != '\n') {
+            raw++;
+        }
+    }
+    char *v = collapse_prose(a, raw, strlen(raw));
+    return v ? strip_surrounding_quotes(v) : NULL;
+}
+
+// Value of `pair` when its key is `want`, else NULL. A YAML block_mapping_pair
+// and a JSON pair both expose key/value fields, so one reader serves both.
+static const char *config_pair_value_if_key(CBMExtractCtx *ctx, TSNode pair, const char *want) {
+    TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key"));
+    TSNode val = ts_node_child_by_field_name(pair, TS_FIELD("value"));
+    if (ts_node_is_null(key) || ts_node_is_null(val)) {
+        return NULL;
+    }
+    char *kt = cbm_node_text(ctx->arena, key, ctx->source);
+    if (!kt || strcmp(strip_surrounding_quotes(kt), want) != 0) {
+        return NULL;
+    }
+    return config_scalar_value(ctx->arena, cbm_node_text(ctx->arena, val, ctx->source));
+}
+
+// First non-empty description value among `container`'s direct pairs, scanned
+// in config_desc_keys priority order.
+static const char *config_container_description(CBMExtractCtx *ctx, TSNode container,
+                                                const char *pair_kind) {
+    uint32_t n = ts_node_named_child_count(container);
+    for (const char *const *k = config_desc_keys; *k; k++) {
+        for (uint32_t i = 0; i < n; i++) {
+            TSNode pair = ts_node_named_child(container, i);
+            if (strcmp(ts_node_type(pair), pair_kind) != 0) {
+                continue;
+            }
+            const char *v = config_pair_value_if_key(ctx, pair, *k);
+            if (v && v[0]) {
+                return v;
+            }
+        }
+    }
+    return NULL;
+}
+
+/* #519 entry point: the file-level description a config document declares about
+ * itself, promoted onto the Module node by cbm_extract_definitions so
+ * nodes_fts.body indexes it and the file is findable by what it SAYS it does,
+ * not only by its path. NULL for every other language. */
+static const char *extract_config_module_description(CBMExtractCtx *ctx) {
+    if (ctx->language == CBM_LANG_YAML) {
+        TSNode bm = find_yaml_toplevel_mapping(ctx->root);
+        return ts_node_is_null(bm) ? NULL
+                                   : config_container_description(ctx, bm, "block_mapping_pair");
+    }
+    if (ctx->language == CBM_LANG_JSON) {
+        TSNode obj = find_json_toplevel_object(ctx->root);
+        return ts_node_is_null(obj) ? NULL : config_container_description(ctx, obj, "pair");
+    }
+    return NULL;
+}
+
+// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
+// Helm values.yaml so each chart's tunables surface as a handful of structured
+// Variables instead of one node per nested leaf (#338).
+static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
+    CBMArena *a = ctx->arena;
+    TSNode bm = find_yaml_toplevel_mapping(root);
     if (ts_node_is_null(bm)) {
         return;
     }
@@ -7587,6 +7837,8 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
     mod.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
     mod.is_exported = true;
     mod.is_test = ctx->result->is_test_file;
+    // #519: index what a config file declares itself to be, not only its path.
+    mod.docstring = extract_config_module_description(ctx);
     cbm_defs_push(&ctx->result->defs, a, mod);
 
     cbm_extract_definitions_without_module(ctx);