From 519c765e1fd67ec12127ab41ec5f231401900ba8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 01:37:27 +0200 Subject: [PATCH 1/6] fix(python): retain typed fields across file boundaries Carry declared instance-field types through extraction and result compaction into the shared Python type registry so chained calls through imported classes resolve their receiver types. Refs #1277 Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 5 + internal/cbm/cbm.h | 19 ++++ internal/cbm/extract_defs.c | 167 ++++++++++++++++++++++++++++++++++ internal/cbm/lsp/py_lsp.c | 42 +++++++++ internal/cbm/result_compact.c | 7 ++ src/pipeline/pass_lsp_cross.c | 130 ++++++++++++++++++++++++++ src/pipeline/pipeline.c | 6 +- tests/test_extraction.c | 57 ++++++++++++ tests/test_pipeline.c | 93 +++++++++++++++++++ tests/test_py_lsp.c | 115 +++++++++++++++++++++++ 10 files changed, 639 insertions(+), 2 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 082bca1d6d..4d01d0c691 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -243,6 +243,11 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) arr->items[arr->count++] = ta; } +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = ft; +} + void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr) { GROW_ARRAY(arr, a); arr->items[arr->count++] = sr; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index fb5e446a5d..7563fa6b59 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -405,6 +405,17 @@ typedef struct { CBMChannelDirection direction; } CBMChannel; +/* Python: one annotated instance field of a class -- `x: T` or `x: T = v` in + * the class body, `self.x: T = v` in __init__, or `self.x = p` where `p` is an + * annotated __init__ parameter. Not a graph node: it only carries the field's + * declared type to the cross-file LSP, so `obj.x.m()` on a class imported + * from another file can be typed (#1277). */ +typedef struct { + const char *class_qn; // QN of the owning class + const char *field_name; // attribute name + const char *type_text; // raw annotation text, resolved later per file +} CBMFieldType; + // Rust: impl Trait for Struct typedef struct { const char *trait_name; // trait name (raw text) @@ -518,6 +529,12 @@ typedef struct { int cap; } CBMChannelArray; +typedef struct { + CBMFieldType *items; + int count; + int cap; +} CBMFieldTypeArray; + // Full extraction result for one file. typedef struct CBMFileResult { CBMArena arena; // owns local memory; composites may also retain child arenas below @@ -536,6 +553,7 @@ typedef struct CBMFileResult { CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation + CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) const char *module_qn; // module qualified name const char *namespace_name; // declared namespace/package (Java/Kotlin/C#/PHP), NULL if none @@ -892,6 +910,7 @@ void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw); void cbm_typerefs_push(CBMTypeRefArray *arr, CBMArena *a, CBMTypeRef tr); void cbm_envaccess_push(CBMEnvAccessArray *arr, CBMArena *a, CBMEnvAccess ea); void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta); +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index bb56e2870f..de5b2e90dd 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -4584,6 +4584,169 @@ static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const cha return true; } +/* ── Python annotated instance fields (#1277) ───────────────────── + * Exported as result->field_types (never graph nodes) so the cross-file LSP + * can type `obj.x` when obj's class lives in another file. Only DECLARED + * types count: a class-body annotation, a `self.x: T = v` annotation in + * __init__, or `self.x = p` where p is an annotated __init__ parameter. An + * unannotated right-hand side is never guessed at. */ + +static void py_push_field_type(CBMExtractCtx *ctx, const char *class_qn, TSNode name_node, + const char *type_text) { + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (!name || !name[0] || !type_text || !type_text[0]) { + return; + } + CBMFieldType ft = {.class_qn = class_qn, .field_name = name, .type_text = type_text}; + cbm_fieldtype_push(&ctx->result->field_types, ctx->arena, ft); +} + +/* The assignment inside an expression_statement, or a null node. */ +static TSNode py_statement_assignment(TSNode stmt) { + TSNode null_node = {0}; + if (strcmp(ts_node_type(stmt), "expression_statement") != 0 || + ts_node_named_child_count(stmt) == 0) { + return null_node; + } + TSNode inner = ts_node_named_child(stmt, 0); + return strcmp(ts_node_type(inner), "assignment") == 0 ? inner : null_node; +} + +/* Annotation text of the __init__ parameter called `name`, or NULL. */ +static const char *py_init_param_annotation(CBMExtractCtx *ctx, TSNode params, const char *name) { + uint32_t n = ts_node_named_child_count(params); + for (uint32_t i = 0; i < n; i++) { + TSNode p = ts_node_named_child(params, i); + const char *pk = ts_node_type(p); + TSNode pname = {0}; + if (strcmp(pk, "typed_default_parameter") == 0) { + pname = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(pk, "typed_parameter") == 0 && ts_node_named_child_count(p) > 0) { + pname = ts_node_named_child(p, 0); + } else { + continue; + } + TSNode ptype = ts_node_child_by_field_name(p, TS_FIELD("type")); + if (ts_node_is_null(pname) || ts_node_is_null(ptype) || + strcmp(ts_node_type(pname), "identifier") != 0) { + continue; + } + char *pn = cbm_node_text(ctx->arena, pname, ctx->source); + if (pn && strcmp(pn, name) == 0) { + return cbm_node_text(ctx->arena, ptype, ctx->source); + } + } + return NULL; +} + +/* `self.x: T = v` / `self.x = p` inside __init__ (self = its first parameter). */ +static void py_init_field_assignment(CBMExtractCtx *ctx, const char *class_qn, TSNode assign, + TSNode params, const char *self_name) { + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + if (ts_node_is_null(left) || strcmp(ts_node_type(left), "attribute") != 0) { + return; + } + TSNode obj = ts_node_child_by_field_name(left, TS_FIELD("object")); + TSNode attr = ts_node_child_by_field_name(left, TS_FIELD("attribute")); + if (ts_node_is_null(obj) || ts_node_is_null(attr) || + strcmp(ts_node_type(obj), "identifier") != 0) { + return; + } + char *obj_name = cbm_node_text(ctx->arena, obj, ctx->source); + if (!obj_name || strcmp(obj_name, self_name) != 0) { + return; + } + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(ann)) { + py_push_field_type(ctx, class_qn, attr, cbm_node_text(ctx->arena, ann, ctx->source)); + return; + } + TSNode right = ts_node_child_by_field_name(assign, TS_FIELD("right")); + if (ts_node_is_null(right) || strcmp(ts_node_type(right), "identifier") != 0) { + return; + } + char *rhs = cbm_node_text(ctx->arena, right, ctx->source); + if (rhs && strcmp(rhs, self_name) != 0) { + py_push_field_type(ctx, class_qn, attr, py_init_param_annotation(ctx, params, rhs)); + } +} + +/* Every `self.x` field assignment in one __init__ body, nested blocks + * included; nested functions, lambdas and classes have their own `self`. */ +static void py_extract_init_fields(CBMExtractCtx *ctx, const char *class_qn, TSNode fn) { + TSNode params = ts_node_child_by_field_name(fn, TS_FIELD("parameters")); + TSNode body = ts_node_child_by_field_name(fn, TS_FIELD("body")); + if (ts_node_is_null(params) || ts_node_is_null(body) || + ts_node_named_child_count(params) == 0) { + return; + } + TSNode self_node = ts_node_named_child(params, 0); + if (strcmp(ts_node_type(self_node), "identifier") != 0) { + return; + } + char *self_name = cbm_node_text(ctx->arena, self_node, ctx->source); + if (!self_name || !self_name[0]) { + return; + } + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_32); + ts_nstack_push(&stack, body); + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); + const char *k = ts_node_type(n); + if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || + strcmp(k, "lambda") == 0) { + continue; + } + TSNode assign = py_statement_assignment(n); + if (!ts_node_is_null(assign)) { + py_init_field_assignment(ctx, class_qn, assign, params, self_name); + continue; + } + /* Reverse push keeps source order on pop. */ + for (uint32_t i = ts_node_named_child_count(n); i > 0; i--) { + ts_nstack_push(&stack, ts_node_named_child(n, i - 1)); + } + } +} + +static void extract_py_field_types(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn) { + TSNode body = ts_node_child_by_field_name(class_node, TS_FIELD("body")); + if (ts_node_is_null(body)) { + return; + } + uint32_t count = ts_node_named_child_count(body); + /* Class-body annotations first: `x: T` / `x: T = v`. */ + for (uint32_t i = 0; i < count; i++) { + TSNode assign = py_statement_assignment(ts_node_named_child(body, i)); + if (ts_node_is_null(assign)) { + continue; + } + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(left) && !ts_node_is_null(ann) && + strcmp(ts_node_type(left), "identifier") == 0) { + py_push_field_type(ctx, class_qn, left, cbm_node_text(ctx->arena, ann, ctx->source)); + } + } + /* Then __init__. */ + for (uint32_t i = 0; i < count; i++) { + TSNode fn = ts_node_named_child(body, i); + if (strcmp(ts_node_type(fn), "decorated_definition") == 0) { + fn = ts_node_child_by_field_name(fn, TS_FIELD("definition")); + } + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "function_definition") != 0) { + continue; + } + TSNode fname = ts_node_child_by_field_name(fn, TS_FIELD("name")); + char *fn_name = + ts_node_is_null(fname) ? NULL : cbm_node_text(ctx->arena, fname, ctx->source); + if (fn_name && strcmp(fn_name, "__init__") == 0) { + py_extract_init_fields(ctx, class_qn, fn); + } + } +} + static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; const char *kind = ts_node_type(node); @@ -4970,6 +5133,10 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // Extract class-level variables (field declarations) extract_class_variables(ctx, node, class_qn, spec); + if (ctx->language == CBM_LANG_PYTHON) { + extract_py_field_types(ctx, node, class_qn); + } + // C# 12 primary-constructor parameters: declared on the class line // (`class Foo(IBar bar, IBaz baz) : Base { ... }`) and bound to implicit // captured fields accessible from any instance member. Tree-sitter c-sharp diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 2cf66ad012..70de5dce78 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -4965,6 +4965,47 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) { return out; } +/* Split a class def's field_defs ("name:QN|name:QN", built by the Python + * field fold in pass_lsp_cross.c) into the parallel NULL-terminated arrays a + * CBMRegisteredType carries. The types are already project QNs, so each one + * becomes a NAMED type verbatim. Without this, `obj.field.method()` on a class + * from another file lost the field's type (#1277); same-file classes get their + * fields from the per-file walk instead. Pure string work: no registry lookup + * happens here, so it is safe inside the type-registration pass. */ +static void py_split_field_defs(CBMArena *arena, const char *field_defs, const char ***names_out, + const CBMType ***types_out) { + *names_out = NULL; + *types_out = NULL; + const char **pairs = py_split_pipe(arena, field_defs); + if (!pairs) + return; + int n = 0; + while (pairs[n]) + n++; + const char **names = + (const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *)); + const CBMType **types = + (const CBMType **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const CBMType *)); + if (!names || !types) + return; + int kept = 0; + for (int i = 0; i < n; i++) { + char *colon = strchr(pairs[i], ':'); + if (!colon || colon == pairs[i] || !colon[1]) + continue; + *colon = '\0'; /* pairs[i] is this call's own arena copy */ + names[kept] = pairs[i]; + types[kept] = cbm_type_named(arena, colon + 1); + kept++; + } + if (kept == 0) + return; + names[kept] = NULL; + types[kept] = NULL; + *names_out = names; + *types_out = types; +} + /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, @@ -4990,6 +5031,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe if (d->method_names_str && d->method_names_str[0]) { rt.method_names = py_split_pipe(arena, d->method_names_str); } + py_split_field_defs(arena, d->field_defs, &rt.field_names, &rt.field_types); cbm_registry_add_type(reg, rt); } } diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 90c8c6d820..82f661d053 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -377,6 +377,12 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->channels.items[i].transport); cr_str(c, &r->channels.items[i].enclosing_func_qn); } + cr_array(c, (void **)&r->field_types.items, r->field_types.count, sizeof(CBMFieldType)); + for (int i = 0; i < r->field_types.count && r->field_types.items; i++) { + cr_str(c, &r->field_types.items[i].class_qn); + cr_str(c, &r->field_types.items[i].field_name); + cr_str(c, &r->field_types.items[i].type_text); + } cr_str(c, &r->module_qn); cr_str(c, &r->namespace_name); cr_list(c, &r->exports); @@ -523,6 +529,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.string_refs.cap = tmp.string_refs.count; tmp.infra_bindings.cap = tmp.infra_bindings.count; tmp.channels.cap = tmp.channels.count; + tmp.field_types.cap = tmp.field_types.count; /* A composite kept its per-unit results only so shallow-copied strings * stayed valid; every string is now a copy of its own. */ diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 3b74cc1ebb..66751a6829 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -529,6 +529,132 @@ static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *resu } } +/* Python annotation text -> the one type name it declares, or NULL. Strips a + * string-literal forward reference and unwraps `Optional[X]`, `X | None` and + * `None | X`; anything else with brackets, commas or a real union (generics, + * Callable, Union[A, B]) names no single receiver type and is dropped. */ +static const char *pxc_py_annotation_type_name(CBMArena *arena, const char *text) { + if (!text) { + return NULL; + } + while (*text == ' ') { + text++; + } + size_t n = strlen(text); + while (n > 0 && text[n - 1] == ' ') { + n--; + } + if (n >= 2 && (text[0] == '"' || text[0] == '\'') && text[n - 1] == text[0]) { + text++; + n -= 2; + } + static const char *const optional_prefixes[] = {"Optional[", "typing.Optional["}; + for (size_t i = 0; i < sizeof(optional_prefixes) / sizeof(optional_prefixes[0]); i++) { + size_t plen = strlen(optional_prefixes[i]); + if (n > plen + 1 && strncmp(text, optional_prefixes[i], plen) == 0 && text[n - 1] == ']') { + text += plen; + n -= plen + 1; + break; + } + } + static const char none_tail[] = " | None"; + static const char none_head[] = "None | "; + const size_t none_len = sizeof(none_tail) - 1; + if (n > none_len && strncmp(text + n - none_len, none_tail, none_len) == 0) { + n -= none_len; + } else if (n > none_len && strncmp(text, none_head, none_len) == 0) { + text += none_len; + n -= none_len; + } + if (n == 0) { + return NULL; + } + for (size_t i = 0; i < n; i++) { + char c = text[i]; + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '.'; + if (!ok) { + return NULL; + } + } + return cbm_arena_strndup(arena, text, n); +} + +/* Python: fold the file's annotated instance fields (result->field_types, + * #1277) into their class defs' field_defs as "name:QN|name:QN", the format + * py_register_lsp_defs reads. Each annotation resolves through the same + * registry + import map, and the same weak-strategy and type-like vetoes, as + * the file's base classes (pxc_resolve_base_qn): a field whose type is not a + * confidently-known project type is left out rather than guessed. Linear in + * the file: one hash of the file's class defs, one pass over its fields. */ +typedef struct { + int owner; /* index into defs[], or -1 */ + const char *type_qn; +} pxc_py_field_t; + +static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs, + int start, int end, const cbm_registry_t *reg, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + int nf = result ? result->field_types.count : 0; + if (!arena || nf == 0 || !defs || start >= end || !reg) { + return; + } + CBMHashTable *by_qn = cbm_ht_create((uint32_t)(end - start)); + pxc_py_field_t *fields = (pxc_py_field_t *)cbm_arena_alloc(arena, (size_t)nf * sizeof(*fields)); + /* Per class def: exact buffer size, then the write cursor into it. */ + size_t *lens = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + size_t *used = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + if (!by_qn || !fields || !lens || !used) { + cbm_ht_free(by_qn); /* NULL-safe */ + return; + } + memset(lens, 0, (size_t)(end - start) * sizeof(size_t)); + memset(used, 0, (size_t)(end - start) * sizeof(size_t)); + for (int si = start; si < end; si++) { + if (defs[si].label && strcmp(defs[si].label, "Class") == 0 && defs[si].qualified_name) { + cbm_ht_set(by_qn, defs[si].qualified_name, &defs[si]); + } + } + for (int f = 0; f < nf; f++) { + const CBMFieldType *ft = &result->field_types.items[f]; + fields[f].owner = -1; + fields[f].type_qn = NULL; + CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; + const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; + const char *qn = + name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + if (!qn || !ft->field_name || !ft->field_name[0]) { + continue; + } + fields[f].owner = (int)(owner - defs); + fields[f].type_qn = qn; + /* "name:QN" plus one byte for the '|' or the NUL after it. */ + lens[fields[f].owner - start] += strlen(ft->field_name) + 1 + strlen(qn) + 1; + } + cbm_ht_free(by_qn); + for (int si = start; si < end; si++) { + char *buf = lens[si - start] ? (char *)cbm_arena_alloc(arena, lens[si - start]) : NULL; + if (buf) { + buf[0] = '\0'; + } + defs[si].field_defs = buf; + } + for (int f = 0; f < nf; f++) { + char *buf = fields[f].owner >= 0 ? (char *)defs[fields[f].owner].field_defs : NULL; + if (!buf) { + continue; + } + int slot = fields[f].owner - start; + int wrote = + snprintf(buf + used[slot], lens[slot] - used[slot], "%s%s:%s", used[slot] ? "|" : "", + result->field_types.items[f].field_name, fields[f].type_qn); + if (wrote > 0) { + used[slot] += (size_t)wrote; + } + } +} + /* Carry one Rust type-level impl independently of any method definition. * `impl Trait for Type {}` is semantically meaningful even when the block is * empty (the trait may provide defaults), so attaching the relation only to @@ -634,6 +760,10 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are idx++; } } + if (files[fi].language == CBM_LANG_PYTHON) { + pxc_fold_py_field_types(arena, fr, defs, file_start, idx, base_reg, def_modules[fi], + imp_keys, imp_vals, imp_count); + } cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index ba41b877c8..32f9df6a52 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1156,7 +1156,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.count * sizeof(CBMStringRef) + (size_t)r->impl_traits.count * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) + - (size_t)r->channels.count * sizeof(CBMChannel); + (size_t)r->channels.count * sizeof(CBMChannel) + + (size_t)r->field_types.count * sizeof(CBMFieldType); cap_other += (size_t)r->imports.cap * sizeof(CBMImport) + (size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) + (size_t)r->throws.cap * sizeof(CBMThrow) + @@ -1165,7 +1166,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.cap * sizeof(CBMStringRef) + (size_t)r->impl_traits.cap * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) + - (size_t)r->channels.cap * sizeof(CBMChannel); + (size_t)r->channels.cap * sizeof(CBMChannel) + + (size_t)r->field_types.cap * sizeof(CBMFieldType); n_defs += (size_t)r->defs.count; n_calls += (size_t)r->calls.count; n_usages += (size_t)r->usages.count; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0b6800cd1f..4980426a92 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1182,6 +1182,62 @@ TEST(cpp_method_return_type_preserves_pointer_and_qualifier) { * ═══════════════════════════════════════════════════════════════════ */ /* --- Python --- */ +/* #1277: annotated Python instance fields reach result->field_types (the + * cross-file LSP's only source for them); nothing unannotated is guessed and + * no graph def is minted for a field. */ +static const char *py_field_type_of(const CBMFileResult *r, const char *cls_suffix, + const char *field) { + for (int i = 0; i < r->field_types.count; i++) { + const CBMFieldType *ft = &r->field_types.items[i]; + size_t cl = strlen(ft->class_qn); + size_t sl = strlen(cls_suffix); + if (cl >= sl && strcmp(ft->class_qn + cl - sl, cls_suffix) == 0 && + strcmp(ft->field_name, field) == 0) { + return ft->type_text; + } + } + return NULL; +} + +TEST(python_annotated_instance_fields_exported_issue1277) { + const char *src = "class Trainer(Base):\n" + " engine: Contract\n" + " limit: int = 3\n" + " plain = 5\n\n" + " def __init__(self, strategies: Contract, n, opt: 'Other' = None):\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.count = n\n" + " self.made = make()\n" + " self.opt = opt\n" + " if n:\n" + " self.nested: Nested = n\n" + " def inner(other: Inner):\n" + " self.hidden = other\n\n" + " def later(self, x: Later):\n" + " self.late = x\n"; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "trainer.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "engine"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "limit"), "int"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "strategies"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "typed"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "opt"), "'Other'"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "nested"), "Nested"); + /* Controls: no declared type, a nested function's binding, a non-__init__ method. */ + ASSERT_NULL(py_field_type_of(r, ".Trainer", "plain")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "count")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "made")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "hidden")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "late")); + ASSERT_EQ(r->field_types.count, 6); + for (int i = 0; i < r->defs.count; i++) { + ASSERT_FALSE(r->defs.items[i].label && strcmp(r->defs.items[i].label, "Field") == 0); + } + cbm_free_result(r); + PASS(); +} + TEST(python_function) { CBMFileResult *r = extract( "def greet(name):\n return f\"Hello {name}\"\n\ndef main():\n greet(\"World\")\n", @@ -8991,6 +9047,7 @@ SUITE(extraction) { /* Scripting */ RUN_TEST(python_function); RUN_TEST(python_class); + RUN_TEST(python_annotated_instance_fields_exported_issue1277); RUN_TEST(js_function); RUN_TEST(js_class); RUN_TEST(ts_function); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 56be6b88e3..b0ebc9f5a2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8166,6 +8166,98 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* #1277: typed instance fields of a Python class imported from another file. + * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an + * LSP strategy. */ +static int count_lsp_calls(cbm_store_t *s, const char *project, const char *caller_suffix, + const char *callee_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + size_t cs = strlen(caller_suffix); + size_t ts = strlen(callee_suffix); + if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0 && + edges[i].properties_json && strstr(edges[i].properties_json, "lsp")) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return hits; +} + +TEST(python_crossfile_typed_field_calls_issue1277) { + const char *contracts = "class Contract:\n" + " def process_batch(self) -> None:\n" + " ...\n"; + const char *trainer = "from contracts import Contract\n\n" + "def make():\n" + " return None\n\n" + "class Trainer:\n" + " engine: Contract\n\n" + " def __init__(self, strategies: Contract) -> None:\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.plain = make()\n"; + const char *loop = "from trainer import Trainer\n\n" + "def run(trainer: Trainer) -> None:\n" + " strategies = trainer.strategies\n" + " strategies.process_batch()\n\n" + "def run_typed(trainer: Trainer) -> None:\n" + " trainer.typed.process_batch()\n\n" + "def run_classlevel(trainer: Trainer) -> None:\n" + " trainer.engine.process_batch()\n\n" + "def run_untyped(trainer: Trainer) -> None:\n" + " trainer.plain.process_batch()\n"; + if (setup_usages_repo("contracts.py", contracts, "trainer.py", trainer) != 0) { + FAIL("failed to create temp dir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs(loop, f); + fclose(f); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); + cbm_pipeline_t *p = cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + + int via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + int via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + int via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + int via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_usages_repo(); + + ASSERT_EQ(via_alias, 1); /* self.x = */ + ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ + ASSERT_EQ(via_class, 1); /* class-level x: T */ + ASSERT_EQ(via_untyped, 0); /* control: no annotation, no guess */ + PASS(); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -16378,6 +16470,7 @@ SUITE(pipeline) { RUN_TEST(implements_creates_override); RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ + RUN_TEST(python_crossfile_typed_field_calls_issue1277); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee519..d65341900d 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -580,6 +580,119 @@ TEST(pylsp_crossfile_method_dispatch) { PASS(); } +/* Issue #1277 (reproducer contributed by @Enferlain): a method call through a + * typed instance field of a class imported from ANOTHER file. The field type + * reaches the resolver only through CBMLSPDef.field_defs on the class def. */ +TEST(pylsp_crossfile_receiver_through_typed_field) { + const char *source = "from trainer import Trainer\n" + "def run(trainer: Trainer):\n" + " strategies = trainer.strategies\n" + " return strategies.process_batch()\n"; + + CBMLSPDef defs[3]; + memset(defs, 0, sizeof(defs)); + + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract"; + + const char *imp_names[] = {"Trainer"}; + const char *imp_qns[] = {"trainer.Trainer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.loop", defs, 3, imp_names, + imp_qns, 1, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "run", "process_batch"), 0); + cbm_arena_destroy(&arena); + PASS(); +} + +/* #1277 on the production path: the sealed shared Tier-2 registry. Covers the + * direct chain, the local alias, a field inherited from a base class, and two + * controls: an undeclared field and a field whose type has no such method + * must stay unresolved (never guessed). */ +TEST(pylsp_crossfile_typed_field_shared_registry) { + CBMArena arena; + cbm_arena_init(&arena); + + CBMLSPDef defs[5]; + memset(defs, 0, sizeof(defs)); + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract|name:builtins.str"; + defs[3].qualified_name = "trainer.SubTrainer"; + defs[3].short_name = "SubTrainer"; + defs[3].label = "Class"; + defs[3].def_module_qn = "trainer"; + defs[3].embedded_types = "trainer.Trainer"; + defs[4].qualified_name = "trainer.Other"; + defs[4].short_name = "Other"; + defs[4].label = "Class"; + defs[4].def_module_qn = "trainer"; + for (int i = 0; i < 5; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 5); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + + const char *src = "from trainer import Trainer, SubTrainer, Other\n" + "def direct(t: Trainer):\n" + " return t.strategies.process_batch()\n" + "def alias(t: Trainer):\n" + " s = t.strategies\n" + " return s.process_batch()\n" + "def inherited(t: SubTrainer):\n" + " return t.strategies.process_batch()\n" + "def undeclared(t: Other):\n" + " return t.strategies.process_batch()\n" + "def wrong_type(t: Trainer):\n" + " return t.name.process_batch()\n"; + const char *imp_names[] = {"Trainer", "SubTrainer", "Other"}; + const char *imp_qns[] = {"trainer.Trainer", "trainer.SubTrainer", "trainer.Other"}; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test.loop", reg, imp_names, + imp_qns, 3, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "direct", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "alias", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "inherited", "Contract.process_batch"), 0); + ASSERT_EQ(find_resolved_arr(&out, "undeclared", "process_batch"), -1); + ASSERT_EQ(find_resolved_arr(&out, "wrong_type", "process_batch"), -1); + + cbm_arena_destroy(&arena); + PASS(); +} + /* Graph-quality guard for the Python seal fix (per-file field overlay). In the * FUSED path the shared Tier-2 registry is read_only, so `self.b = Bar()` can no * longer be recorded by mutating the shared registry; it is recorded in the per-file @@ -2193,6 +2306,8 @@ SUITE(py_lsp) { RUN_TEST(pylsp_pep695_generic_class); /* Phase 9 — cross-file + batch */ RUN_TEST(pylsp_crossfile_method_dispatch); + RUN_TEST(pylsp_crossfile_receiver_through_typed_field); + RUN_TEST(pylsp_crossfile_typed_field_shared_registry); RUN_TEST(pylsp_fused_self_attr_chain_via_overlay); RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228); RUN_TEST(pylsp_crossfile_inheritance); From bf2daa36036745b6b4bac5b118cd2ba778efa30c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 18:25:03 +0200 Subject: [PATCH 2/6] fix(lsp): detach field arrays before overlay refinement Copy inherited field-type pointer arrays into the writable registry arena before Python refines an existing field. Keep shared type objects intact and avoid retaining per-file scratch types in the sealed fallback. Add deterministic shared-state and later-overlay coverage, retaining the sequential Python field regression and adding a forced-parallel variant. Refs #1277 Signed-off-by: Martin Vogel --- internal/cbm/lsp/type_registry.c | 22 +++++- tests/test_c_lsp.c | 115 +++++++++++++++++++++++++++++++ tests/test_pipeline.c | 86 ++++++++++++++++++----- 3 files changed, 203 insertions(+), 20 deletions(-) diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 4e2f02012e..06b55c391d 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -677,8 +677,28 @@ CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const cha for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { const CBMRegisteredType *base = lookup_type_self(r, qualified_name); if (base) { + CBMRegisteredType copy = *base; + /* Field refinements overwrite entries in this array. The type + * objects remain immutable, but the writable overlay must own + * the pointer array rather than mutate its shared fallback. */ + if (base->field_types) { + size_t count = 0; + /* Types are parallel to names; a named field may have no + * type, so an interior NULL does not terminate that array. */ + while (base->field_names ? base->field_names[count] != NULL + : base->field_types[count] != NULL) { + count++; + } + const CBMType **field_types = (const CBMType **)cbm_arena_alloc( + head->arena, (count + 1) * sizeof(*field_types)); + if (!field_types) { + return NULL; + } + memcpy(field_types, base->field_types, (count + 1) * sizeof(*field_types)); + copy.field_types = field_types; + } int before = head->type_count; - cbm_registry_add_type(head, *base); + cbm_registry_add_type(head, copy); if (head->type_count == before + 1) { return &head->types[before]; } diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index 933cacc36f..d890911506 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -15350,6 +15350,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { memset(&t, 0, sizeof(t)); t.qualified_name = "pkg.T"; t.short_name = "T"; + const char *field_names[] = {"untyped", "typed", NULL}; + const CBMType *field_types[] = {NULL, cbm_type_named(&arena, "pkg.Value"), NULL}; + t.field_names = field_names; + t.field_types = field_types; cbm_registry_add_type(&base, t); cbm_registry_finalize(&base); base.read_only = true; @@ -15414,6 +15418,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T"); ASSERT_NOT_NULL(wt); ASSERT_EQ(overlay.type_count, 1); + /* A missing type for an earlier named field must not truncate the copy. */ + bool field_slots_preserved = wt->field_types && wt->field_types != field_types && + wt->field_types[0] == NULL && + wt->field_types[1] == field_types[1] && wt->field_types[2] == NULL; cbm_registry_all_types_chain(&overlay, &ti); seen = 0; while (cbm_type_short_iter_next(&ti) >= 0) { @@ -15430,6 +15438,7 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { cbm_arena_destroy(&scratch); cbm_arena_destroy(&arena); + ASSERT_TRUE(field_slots_preserved); PASS(); } @@ -15804,6 +15813,111 @@ TEST(seal_py_shared_registry_readonly_fields) { PASS(); } +/* #1277: the production resolver receives a writable overlay, so refining an + * existing field must detach its pointer array from the sealed fallback. */ +TEST(seal_py_overlay_existing_field_keeps_shared_type) { + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef defs[4] = {0}; + defs[0].qualified_name = "test.mod.Foo"; + defs[0].short_name = "Foo"; + defs[0].label = "Class"; + defs[0].def_module_qn = "test.mod"; + defs[0].field_defs = "x:test.contracts.Contract"; + defs[1].qualified_name = "test.mod.Foo.m"; + defs[1].short_name = "m"; + defs[1].label = "Method"; + defs[1].receiver_type = "test.mod.Foo"; + defs[1].def_module_qn = "test.mod"; + defs[2].qualified_name = "test.contracts.Contract"; + defs[2].short_name = "Contract"; + defs[2].label = "Class"; + defs[2].def_module_qn = "test.contracts"; + defs[3].qualified_name = "test.contracts.Contract.process_batch"; + defs[3].short_name = "process_batch"; + defs[3].label = "Method"; + defs[3].receiver_type = "test.contracts.Contract"; + defs[3].def_module_qn = "test.contracts"; + for (int i = 0; i < 4; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 4); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + const CBMRegisteredType *base = cbm_registry_lookup_type(reg, "test.mod.Foo"); + ASSERT_NOT_NULL(base); + ASSERT_NOT_NULL(base->field_names); + ASSERT_NOT_NULL(base->field_types); + const char **names_before = base->field_names; + const CBMType **types_before = base->field_types; + const CBMType *type_before = base->field_types[0]; + ASSERT_NOT_NULL(type_before); + ASSERT_NULL(base->field_types[1]); + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *writer = "class Foo:\n" + " def m(self):\n" + " self.x = 1\n"; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&scratch, writer, (int)strlen(writer), "test.mod", &overlay, + NULL, NULL, 0, NULL, &out, NULL); + + /* Observe while scratch is live, then release both arenas before any + * failure assertion so the regression also fails cleanly under LSan. */ + bool shared_names_unchanged = base->field_names == names_before; + bool shared_types_unchanged = base->field_types == types_before; + bool shared_element_unchanged = base->field_types[0] == type_before; + bool shared_terminator_unchanged = base->field_types[1] == NULL; + const CBMRegisteredType *refined = cbm_registry_lookup_type(&overlay, "test.mod.Foo"); + bool local_refined = refined && refined != base && refined->field_types && + refined->field_types != types_before && refined->field_types[0] && + refined->field_types[0] != type_before && + refined->field_types[0]->kind == CBM_TYPE_BUILTIN && + refined->field_types[1] == NULL; + cbm_arena_destroy(&scratch); + + /* Never read a corrupted shared pointer after scratch destruction. When + * intact, a later file must still see the original Contract field. */ + int hits = 0; + if (shared_names_unchanged && shared_types_unchanged && shared_element_unchanged && + shared_terminator_unchanged && local_refined) { + cbm_arena_init(&scratch); + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *reader = "from mod import Foo\n\n" + "def run(foo: Foo):\n" + " foo.x.process_batch()\n"; + const char *import_names[] = {"Foo"}; + const char *import_qns[] = {"test.mod.Foo"}; + memset(&out, 0, sizeof(out)); + cbm_run_py_lsp_cross_with_registry(&scratch, reader, (int)strlen(reader), "test.reader", + &overlay, import_names, import_qns, 1, NULL, &out, NULL); + for (int i = 0; i < out.count; i++) { + const CBMResolvedCall *call = &out.items[i]; + if (call->caller_qn && call->callee_qn && + strcmp(call->caller_qn, "test.reader.run") == 0 && + strcmp(call->callee_qn, "test.contracts.Contract.process_batch") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && + call->confidence >= CBM_LSP_CONFIDENCE_FLOOR) { + hits++; + } + } + cbm_arena_destroy(&scratch); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(shared_names_unchanged); + ASSERT_TRUE(shared_types_unchanged); + ASSERT_TRUE(shared_element_unchanged); + ASSERT_TRUE(shared_terminator_unchanged); + ASSERT_TRUE(local_refined); + ASSERT_EQ(hits, 1); + PASS(); +} + TEST(seal_cs_shared_registry_readonly) { CBMArena arena; cbm_arena_init(&arena); @@ -16402,6 +16516,7 @@ SUITE(c_lsp) { RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write); RUN_TEST(clsp_method_return_refinement_is_copy_on_write); RUN_TEST(seal_py_shared_registry_readonly_fields); + RUN_TEST(seal_py_overlay_existing_field_keeps_shared_type); RUN_TEST(seal_cs_shared_registry_readonly); RUN_TEST(seal_ts_shared_registry_readonly); RUN_TEST(seal_go_shared_registry_readonly); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index b0ebc9f5a2..9e26829e27 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8198,7 +8198,7 @@ static int count_lsp_calls(cbm_store_t *s, const char *project, const char *call return hits; } -TEST(python_crossfile_typed_field_calls_issue1277) { +static int check_python_crossfile_typed_field_calls_issue1277(bool parallel) { const char *contracts = "class Contract:\n" " def process_batch(self) -> None:\n" " ...\n"; @@ -8226,31 +8226,70 @@ TEST(python_crossfile_typed_field_calls_issue1277) { } char path[512]; snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); - FILE *f = fopen(path, "w"); - ASSERT_NOT_NULL(f); - fputs(loop, f); - fclose(f); + if (th_write_file(path, loop) != 0) { + teardown_usages_repo(); + FAIL("failed to write loop.py"); + } + if (parallel) { + /* Three semantic files plus 50 inert files exceed the production + * threshold for the fused parallel pipeline. */ + for (int i = 0; i < 50; i++) { + snprintf(path, sizeof(path), "%s/field_pad_%02d.py", g_usages_tmpdir, i); + if (th_write_file(path, "field_padding = 0\n") != 0) { + teardown_usages_repo(); + FAIL("failed to write parallel-selection fixture"); + } + } + } + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + FAIL("failed to save pipeline environment"); + } + int pin_workers_rc = parallel ? cbm_setenv("CBM_WORKERS", "4", 1) : 0; + int pin_single_rc = parallel ? cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") : 0; char db_path[512]; snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); - cbm_pipeline_t *p = cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL); - ASSERT_NOT_NULL(p); - ASSERT_EQ(cbm_pipeline_run(p), 0); - cbm_store_t *s = cbm_store_open_path(db_path); - ASSERT_NOT_NULL(s); - const char *project = cbm_pipeline_project_name(p); - - int via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); - int via_typed = - count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); - int via_class = - count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); - int via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_pipeline_t *p = pin_workers_rc == 0 && pin_single_rc == 0 + ? cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL) + : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; + bool store_opened = s != NULL; + int via_alias = -1, via_typed = -1, via_class = -1, via_untyped = -1; + if (s) { + const char *project = cbm_pipeline_project_name(p); + via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_store_close(s); + } - cbm_store_close(s); cbm_pipeline_free(p); + int restore_workers_rc = + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + int restore_single_rc = saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); teardown_usages_repo(); + ASSERT_EQ(pin_workers_rc, 0); + ASSERT_EQ(pin_single_rc, 0); + ASSERT_EQ(restore_workers_rc, 0); + ASSERT_EQ(restore_single_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_TRUE(store_opened); ASSERT_EQ(via_alias, 1); /* self.x = */ ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ ASSERT_EQ(via_class, 1); /* class-level x: T */ @@ -8258,6 +8297,14 @@ TEST(python_crossfile_typed_field_calls_issue1277) { PASS(); } +TEST(python_crossfile_typed_field_calls_issue1277) { + return check_python_crossfile_typed_field_calls_issue1277(false); +} + +TEST(python_crossfile_typed_field_calls_issue1277_parallel) { + return check_python_crossfile_typed_field_calls_issue1277(true); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -16471,6 +16518,7 @@ SUITE(pipeline) { RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ RUN_TEST(python_crossfile_typed_field_calls_issue1277); + RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); From e3535dc8a80a50120a2be48e07b05dd3e2dec544 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 20:11:45 +0200 Subject: [PATCH 3/6] fix(lsp): bound repeated resolver work without truncating large files Memoize C expression evaluation, index repeated scope and registry queries, and report incomplete walks explicitly on memo or depth failure. Refs #1527 Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 203 +++++++++++-- internal/cbm/cbm.h | 15 + internal/cbm/lsp/c_lsp.c | 404 +++++++++++++++++++++++--- internal/cbm/lsp/c_lsp.h | 58 +++- internal/cbm/lsp/kotlin_lsp.c | 9 +- internal/cbm/lsp/lsp_node_iter.h | 29 ++ internal/cbm/lsp/lsp_work.h | 24 ++ internal/cbm/lsp/py_lsp.c | 377 ++++++++++++++++++------ internal/cbm/lsp/py_lsp.h | 41 ++- internal/cbm/lsp/scope.c | 164 +++++++++-- internal/cbm/lsp/scope.h | 35 ++- internal/cbm/lsp/ts_lsp.c | 8 +- internal/cbm/lsp/type_registry.c | 21 +- internal/cbm/lsp/type_registry.h | 7 + src/pipeline/pass_lsp_cross.c | 46 ++- src/pipeline/pass_parallel.c | 38 ++- tests/test_c_lsp.c | 477 +++++++++++++++++++++++++++++++ tests/test_extraction.c | 46 +++ tests/test_pipeline.c | 182 ++++++++++++ tests/test_py_lsp.c | 463 +++++++++++++++++++++++++++++- tests/test_py_lsp_scale.c | 28 +- 21 files changed, 2416 insertions(+), 259 deletions(-) create mode 100644 internal/cbm/lsp/lsp_work.h diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 4d01d0c691..463c44074d 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -20,6 +20,7 @@ #include "lsp/java_lsp.h" #include "lsp/kotlin_lsp.h" #include "lsp/rust_lsp.h" +#include "lsp/lsp_work.h" #include "preprocessor.h" #include "sql_values.h" // #1735: literal INSERT rows kept out of the SQL parse #include "foundation/compat.h" @@ -2146,6 +2147,183 @@ static bool cbm_sql_values_exclusion_on(const char *rel_path) { return true; } +/* ── Innermost enclosing Function/Method per call (#1527) ────────────── + * The call-context metrics attribute each call to the Function/Method def with + * the SMALLEST line span containing the call's line, ties to the lowest def + * index. Scanning every def for every call made that O(calls x defs) per file + * -- quadratic in file size, the largest single cost on a module with + * thousands of methods. The sweep below gives the identical answer in + * O((calls + defs) log defs): calls in line order, candidates entering a + * min-heap keyed (span, index) once their start line is reached and leaving it + * lazily once their end line has passed -- lines only grow, so a def that has + * ended never contains a later call. The heap top is then the smallest + * (span, index) among the defs that contain the line. */ +static bool enclosing_is_callable(const CBMDefinition *d) { + return d->name && d->label && + (strcmp(d->label, "Function") == 0 || strcmp(d->label, "Method") == 0); +} + +/* The reference answer for one line (and the path when the sweep's arrays + * cannot be allocated). */ +static int cbm_enclosing_callable_scan(const CBMFileResult *result, int line) { + int best = -1; + int best_span = -1; + for (int di = 0; di < result->defs.count; di++) { + const CBMDefinition *d = &result->defs.items[di]; + CBM_LSP_WORK(1); + if (!enclosing_is_callable(d)) { + continue; + } + if ((int)d->start_line <= line && line <= (int)d->end_line) { + int span = (int)d->end_line - (int)d->start_line; + if (best < 0 || span < best_span) { + best_span = span; + best = di; + } + } + } + return best; +} + +static CBM_TLS const CBMFileResult *tl_enclosing_result; + +static int enclosing_def_by_start(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + uint32_t la = tl_enclosing_result->defs.items[ia].start_line; + uint32_t lb = tl_enclosing_result->defs.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +static int enclosing_call_by_line(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + int la = tl_enclosing_result->calls.items[ia].start_line; + int lb = tl_enclosing_result->calls.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +/* Heap order: smaller span first, then smaller def index. */ +static bool enclosing_heap_less(const CBMFileResult *r, int a, int b) { + const CBMDefinition *da = &r->defs.items[a]; + const CBMDefinition *db = &r->defs.items[b]; + int sa = (int)da->end_line - (int)da->start_line; + int sb = (int)db->end_line - (int)db->start_line; + return sa != sb ? sa < sb : a < b; +} + +static void enclosing_heap_push(const CBMFileResult *r, int *heap, int *n, int di) { + int i = (*n)++; + heap[i] = di; + while (i > 0) { + int parent = (i - 1) / 2; + CBM_LSP_WORK(1); + if (!enclosing_heap_less(r, heap[i], heap[parent])) { + break; + } + int t = heap[i]; + heap[i] = heap[parent]; + heap[parent] = t; + i = parent; + } +} + +static void enclosing_heap_pop(const CBMFileResult *r, int *heap, int *n) { + heap[0] = heap[--(*n)]; + int i = 0; + for (;;) { + int l = (2 * i) + 1; + int m = i; + CBM_LSP_WORK(1); + if (l < *n && enclosing_heap_less(r, heap[l], heap[m])) { + m = l; + } + if (l + 1 < *n && enclosing_heap_less(r, heap[l + 1], heap[m])) { + m = l + 1; + } + if (m == i) { + break; + } + int t = heap[i]; + heap[i] = heap[m]; + heap[m] = t; + i = m; + } +} + +/* enclosing[ci] = innermost Function/Method def index for call ci, or -1. + * Returns NULL when the working arrays cannot be allocated; the caller then + * asks cbm_enclosing_callable_scan per call (same answer, old cost). */ +static int *cbm_enclosing_callables(const CBMFileResult *result, int call_count) { + int def_count = result->defs.count; + if (call_count <= 0) { + return NULL; + } + size_t call_bytes = (size_t)call_count * sizeof(int); + size_t def_bytes = (size_t)(def_count > 0 ? def_count : 1) * sizeof(int); + int *enclosing = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *call_order = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *cands = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + int *heap = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + if (!enclosing || !call_order || !cands || !heap) { + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return NULL; + } + int ncand = 0; + for (int di = 0; di < def_count; di++) { + if (enclosing_is_callable(&result->defs.items[di])) { + cands[ncand++] = di; + } + } + for (int ci = 0; ci < call_count; ci++) { + call_order[ci] = ci; + enclosing[ci] = -1; + } + tl_enclosing_result = result; + qsort(cands, (size_t)ncand, sizeof(int), enclosing_def_by_start); + qsort(call_order, (size_t)call_count, sizeof(int), enclosing_call_by_line); + tl_enclosing_result = NULL; + + int heap_n = 0; + int next = 0; + for (int k = 0; k < call_count; k++) { + int ci = call_order[k]; + int line = result->calls.items[ci].start_line; + while (next < ncand && (int)result->defs.items[cands[next]].start_line <= line) { + enclosing_heap_push(result, heap, &heap_n, cands[next++]); + } + while (heap_n > 0 && (int)result->defs.items[heap[0]].end_line < line) { + enclosing_heap_pop(result, heap, &heap_n); + } + enclosing[ci] = heap_n > 0 ? heap[0] : -1; + } + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return enclosing; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result) { + int *enclosing = cbm_enclosing_callables(result, result->calls.count); + if (!enclosing) { + return result->calls.count > 0 ? -1 : 0; + } + int mismatches = 0; + for (int ci = 0; ci < result->calls.count; ci++) { + int line = result->calls.items[ci].start_line; + if (line > 0 && enclosing[ci] != cbm_enclosing_callable_scan(result, line)) { + mismatches++; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + return mismatches; +} +#endif + static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, @@ -2509,8 +2687,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Also run LSP on expanded source for additional type-resolved // calls (language is already C/C++/CUDA — checked in enclosing // block). Runs in every mode. - cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, - language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + if (!result->lsp_skipped) { + cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, + language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + } /* All C-LSP emitters stamp origin directly so rewrite-time * comparisons are already safe. Keep this boundary sweep as @@ -2833,28 +3013,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C d->param_count = pc; } + int *enclosing = cbm_enclosing_callables(result, orig_calls_count); for (int ci = 0; ci < orig_calls_count; ci++) { const CBMCall *c = &result->calls.items[ci]; if (!c->callee_name || c->start_line <= 0) { continue; } // Innermost enclosing Function/Method def by line range (smallest span). - int best = -1; - int best_span = -1; - for (int di = 0; di < def_count; di++) { - const CBMDefinition *d = &result->defs.items[di]; - if (!d->name || !d->label || - (strcmp(d->label, "Function") != 0 && strcmp(d->label, "Method") != 0)) { - continue; - } - if ((int)d->start_line <= c->start_line && c->start_line <= (int)d->end_line) { - int span = (int)d->end_line - (int)d->start_line; - if (best < 0 || span < best_span) { - best_span = span; - best = di; - } - } - } + int best = enclosing ? enclosing[ci] : cbm_enclosing_callable_scan(result, c->start_line); if (best < 0) { continue; } @@ -2899,6 +3065,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } free(has_self); free(has_guarded); + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); uint64_t t2 = now_ns(); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 7563fa6b59..6cf6d04fec 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -450,6 +450,14 @@ typedef struct { int cap; } CBMResolvedCallArray; +/* Refinement status: existing parser/no-op exits are separate from these + * explicit evaluator failures. The caller retains prior completed output. */ +typedef enum { + CBM_LSP_COMPLETE = 0, + CBM_LSP_MEMO_FAILED, + CBM_LSP_DEPTH_EXCEEDED, +} CBMLSPStatus; + // Growable arrays used during extraction. typedef struct { CBMDefinition *items; @@ -914,6 +922,13 @@ void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); + +#ifdef CBM_ENABLE_TEST_SEAMS +/* #1527 seam: how many of result's calls get a different innermost enclosing + * Function/Method from the call-context sweep than from the reference + * per-call scan (0 = identical), or -1 when the sweep could not allocate. */ +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result); +#endif void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 74b97d69d9..3506707615 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -1,10 +1,12 @@ #include "c_lsp.h" #include "lsp_node_iter.h" +#include "lsp_work.h" #include "../helpers.h" #include #include #include #include +#include /* Safe kind accessor — returns CBM_TYPE_UNKNOWN for NULL types. * Prevents SEGV in c_eval_expr_type_inner on unusual C++ AST shapes. */ @@ -12,6 +14,47 @@ static inline CBMTypeKind safe_kind(const CBMType *t) { return t ? t->kind : CBM_TYPE_UNKNOWN; } +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int c_test_memo_allocations_left = -1; +static _Thread_local bool c_test_depth_failure; +void cbm_c_lsp_test_depth_fail(bool enabled) { + c_test_depth_failure = enabled; +} +static _Thread_local uint32_t c_test_walks; +uint32_t cbm_c_lsp_test_walks_take(void) { + uint32_t walks = c_test_walks; + c_test_walks = 0; + return walks; +} +void cbm_c_lsp_test_memo_fail_after(int successful_allocations) { + c_test_memo_allocations_left = successful_allocations; +} +#endif + +static void c_memo_test_stage(CLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + c_test_walks++; + const char *depth_stage = getenv("CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 257; + const char *fail_stage = getenv("CBM_TEST_C_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime keeps the reason available even when the arena is full. + * Result compaction/spilling copies strings, including non-arena strings. */ + result->error_msg = + status == CBM_LSP_DEPTH_EXCEEDED ? CBM_C_LSP_DEPTH_ERROR : CBM_C_LSP_MEMO_ERROR; +} + // Forward declarations static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node); static void c_emit_resolved_call(CLSPContext *ctx, const char *callee_qn, const char *strategy, @@ -59,6 +102,11 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc const CBMTypeRegistry *registry, const char *module_qn, bool cpp_mode, CBMResolvedCallArray *out) { memset(ctx, 0, sizeof(CLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = c_test_memo_allocations_left; + if (c_test_depth_failure) + ctx->eval_depth = 257; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; @@ -191,14 +239,7 @@ static bool c_scope_declares_name(const CBMScope *scope, const char *var_name) { if (!scope || !var_name) { return false; } - for (const CBMScopeChunk *chunk = scope->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, var_name) == 0) { - return true; - } - } - } - return false; + return cbm_scope_lookup_local(scope, var_name) != NULL; } /* Return the exact fp-state entry for the nearest lexical declaration. An @@ -1474,26 +1515,129 @@ static const char *type_to_qn(const CBMType *t) { static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node); +/* Stack guard: the evaluator recurses once per expression nesting level. + * There is deliberately NO per-file work budget (#1527): the 10000-step cap + * that used to sit here (#323) left every later expression in a large file + * unresolved while the file still reported status `indexed`. What it was + * containing -- a fluent chain re-evaluating its receiver at every link, 2^N + * evaluations (the Bitcoin versionbits hang) -- is removed at the root by the + * per-evaluation memo below. */ #define C_EVAL_DEPTH_LIMIT 256 -#define C_EVAL_MAX_STEPS_PER_FILE 10000 +enum { C_EVAL_MEMO_INITIAL_CAP = 64 }; + +static uint32_t c_eval_memo_hash(const void *id) { + uintptr_t v = (uintptr_t)id >> 3; /* node structs are >=8-byte aligned */ + return (uint32_t)((v ^ (v >> 29)) * 2654435761u); +} + +/* Empty the memo for a new outermost evaluation: O(slots the last one used). */ +static void c_eval_memo_reset(CLSPContext *ctx) { + for (int i = 0; i < ctx->eval_memo_count; i++) { + ctx->eval_memo[ctx->eval_memo_used[i]].node_id = NULL; + } + ctx->eval_memo_count = 0; +} + +static const CBMType *c_eval_memo_lookup(const CLSPContext *ctx, const void *id) { + if (ctx->eval_memo_cap == 0) { + return NULL; + } + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + for (uint32_t i = c_eval_memo_hash(id) & mask;; i = (i + 1) & mask) { + const struct CLSPEvalMemoSlot *e = &ctx->eval_memo[i]; + if (!e->node_id) { + return NULL; + } + if (e->node_id == id) { + return e->result; + } + } +} + +static void c_eval_memo_place(CLSPContext *ctx, const void *id, const CBMType *result) { + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + uint32_t i = c_eval_memo_hash(id) & mask; + while (ctx->eval_memo[i].node_id) { + i = (i + 1) & mask; + } + ctx->eval_memo[i].node_id = id; + ctx->eval_memo[i].result = result; + ctx->eval_memo_used[ctx->eval_memo_count++] = (int)i; +} + +static void *c_eval_memo_alloc(CLSPContext *ctx, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return NULL; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif + return cbm_arena_alloc(ctx->arena, size); +} + +/* Memoization is required to bound receiver re-evaluation. An allocation + * failure stops this file's refinement and is reported to the caller. */ +static bool c_eval_memo_insert(CLSPContext *ctx, const void *id, const CBMType *result) { + if (ctx->eval_memo_count >= ctx->eval_memo_cap / 2) { + if (ctx->eval_memo_cap > INT_MAX / 2) + return false; + int new_cap = ctx->eval_memo_cap ? ctx->eval_memo_cap * 2 : C_EVAL_MEMO_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(struct CLSPEvalMemoSlot) || + (size_t)(new_cap / 2) > SIZE_MAX / sizeof(int)) + return false; + struct CLSPEvalMemoSlot *slots = c_eval_memo_alloc(ctx, (size_t)new_cap * sizeof(*slots)); + if (!slots) + return false; + int *used = c_eval_memo_alloc(ctx, (size_t)(new_cap / 2) * sizeof(int)); + if (!used) + return false; + memset(slots, 0, (size_t)new_cap * sizeof(*slots)); + struct CLSPEvalMemoSlot *old = ctx->eval_memo; + const int *old_used = ctx->eval_memo_used; + int old_count = ctx->eval_memo_count; + ctx->eval_memo = slots; + ctx->eval_memo_used = used; + ctx->eval_memo_cap = new_cap; + ctx->eval_memo_count = 0; + for (int k = 0; k < old_count; k++) { + const struct CLSPEvalMemoSlot *e = &old[old_used[k]]; + c_eval_memo_place(ctx, e->node_id, e->result); + } + } + c_eval_memo_place(ctx, id, result); + return true; +} const CBMType *c_eval_expr_type(CLSPContext *ctx, TSNode node) { - if (ts_node_is_null(node)) + if (ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); - /* Expression type evaluation is best-effort. Some recovery-mode C++ ASTs - * can repeatedly drive member/type lookup without increasing recursion - * depth. Keep a generous per-file work budget so pathological expressions - * degrade to unknown instead of hanging repository indexing. */ - if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT || ctx->eval_steps++ > C_EVAL_MAX_STEPS_PER_FILE) { - if (ctx->debug && ctx->eval_steps == C_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [clsp] expression eval step budget exhausted; returning unknown\n"); - } + if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT) { + ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } + if (ctx->eval_depth == 0) { + c_eval_memo_reset(ctx); /* scope may have changed since the last evaluation */ + } + const CBMType *cached = c_eval_memo_lookup(ctx, node.id); + if (cached) { + return cached; + } + CBM_LSP_WORK(1); /* one real evaluation (complexity tests) */ + uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; const CBMType *result = c_eval_expr_type_inner(ctx, node); ctx->eval_depth--; - return result ? result : cbm_type_unknown(); + if (!result) { + result = cbm_type_unknown(); + } + /* Never memoize a result a depth cutoff may have truncated: the incomplete walk + * must stop before a second receiver branch repeats the truncated work. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !c_eval_memo_insert(ctx, node.id, result)) { + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + } + return ctx->eval_failure ? cbm_type_unknown() : result; } static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node) { @@ -2963,6 +3107,8 @@ static const CBMType *c_parse_declaration_type(CLSPContext *ctx, TSNode decl_nod } void c_process_statement(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ts_node_is_null(node)) return; const char *kind = ts_node_type(node); @@ -3930,6 +4076,8 @@ static void c_resolve_calls_in_node_inner(CLSPContext *ctx, TSNode node); * frames via c_adl_resolve). Past the cap the subtree is skipped — its calls * stay unresolved, which is graceful degradation, not a crash. */ static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ctx->walk_depth >= C_LSP_MAX_WALK_DEPTH) return; ctx->walk_depth++; @@ -4783,6 +4931,8 @@ recurse:; // ============================================================================ static void c_process_function(CLSPContext *ctx, TSNode func_node) { + if (ctx->eval_failure) + return; TSNode decl = ts_node_child_by_field_name(func_node, "declarator", 10); if (ts_node_is_null(decl)) return; @@ -4973,6 +5123,8 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // Process a top-level or nested declaration within a namespace/class body, // handling template_declaration wrapping. static void c_process_body_child(CLSPContext *ctx, TSNode child) { + if (ctx->eval_failure) + return; if (ts_node_is_null(child)) return; const char *ck = ts_node_type(child); @@ -5429,6 +5581,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { if (ts_node_is_null(root)) return; + int resolved_before = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + // Collect top-level children once (O(n)); both passes reuse the array. // Indexing ts_node_child(root,i) per iteration is O(i) → O(n²) on a wide root. uint32_t kn = 0; @@ -5437,7 +5591,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { TSNode inner; // Pass 1: process using declarations and global variables - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; const char *ck = ts_node_type(child); @@ -5461,7 +5615,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { } // Pass 2: process functions, namespaces, classes, templates - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; c_process_body_child(ctx, child); } @@ -5469,6 +5623,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { // Release the per-file negative-lookup memo (malloc-owned; only allocated in // shared-registry mode). No-op when it was never populated. c_neg_memo_free(ctx); + if (ctx->eval_failure && ctx->resolved_calls) + ctx->resolved_calls->count = resolved_before; } // ============================================================================ @@ -5682,8 +5838,116 @@ static void c_rewrite_proved_destructor_candidates(CBMFileResult *result) { } } +/* QN -> index of its FIRST registration in reg->types, for the single-file + * registration loop below (#1527). That loop asked "is this type registered + * yet, and where?" by scanning every registered type per Field and per + * Method -- O(defs x types) before the registry was indexed. Tracking the + * first index as types are added answers the same question in O(1); the + * types array may be reallocated as it grows, so indices, not pointers. */ +typedef struct { + const char **qn; /* NULL = empty slot */ + int *first; + int cap; /* power of two, or 0 when unavailable */ + int count; +} CTypeFirstIndex; + +static uint32_t c_type_first_hash(const char *s) { + uint32_t h = 2166136261u; + for (const unsigned char *p = (const unsigned char *)s; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Slot holding qn, or the empty slot where it belongs. */ +static int c_type_first_slot(const CTypeFirstIndex *ix, const char *qn) { + uint32_t mask = (uint32_t)ix->cap - 1; + uint32_t i = c_type_first_hash(qn) & mask; + while (ix->qn[i] && strcmp(ix->qn[i], qn) != 0) { + CBM_LSP_WORK(1); + i = (i + 1) & mask; + } + return (int)i; +} + +static bool c_type_first_alloc(CTypeFirstIndex *ix, CBMArena *arena, int cap) { + const char **qn = (const char **)cbm_arena_alloc(arena, (size_t)cap * sizeof(*qn)); + int *first = (int *)cbm_arena_alloc(arena, (size_t)cap * sizeof(*first)); + if (!qn || !first) { + return false; + } + memset(qn, 0, (size_t)cap * sizeof(*qn)); + ix->qn = qn; + ix->first = first; + ix->cap = cap; + ix->count = 0; + return true; +} + +/* Record reg->types[idx]; an existing QN keeps its first index. On OOM the + * index turns itself off (cap 0) and callers fall back to the scan. */ +static void c_type_first_note(CTypeFirstIndex *ix, CBMArena *arena, const CBMTypeRegistry *reg, + int idx) { + const char *qn = reg->types[idx].qualified_name; + if (ix->cap == 0 || !qn) { + return; + } + if ((ix->count + 1) * 2 > ix->cap) { + CTypeFirstIndex grown = {0}; + if (!c_type_first_alloc(&grown, arena, ix->cap * 2)) { + ix->cap = 0; + return; + } + for (int i = 0; i < ix->cap; i++) { + if (ix->qn[i]) { + int s = c_type_first_slot(&grown, ix->qn[i]); + grown.qn[s] = ix->qn[i]; + grown.first[s] = ix->first[i]; + grown.count++; + } + } + *ix = grown; + } + int s = c_type_first_slot(ix, qn); + if (!ix->qn[s]) { + ix->qn[s] = qn; + ix->first[s] = idx; + ix->count++; + } +} + +/* Index of the first registered type named qn, or -1. Same answer as the + * linear scan it replaces (which also serves as the OOM fallback). */ +static int c_type_first_find(const CTypeFirstIndex *ix, const CBMTypeRegistry *reg, + const char *qn) { + if (ix->cap > 0) { + int s = c_type_first_slot(ix, qn); + return ix->qn[s] ? ix->first[s] : -1; + } + for (int ri = 0; ri < reg->type_count; ri++) { + CBM_LSP_WORK(1); + if (strcmp(reg->types[ri].qualified_name, qn) == 0) { + return ri; + } + } + return -1; +} + +/* cbm_registry_add_type plus the first-index bookkeeping. */ +static void c_add_type_indexed(CBMTypeRegistry *reg, CTypeFirstIndex *ix, CBMArena *arena, + CBMRegisteredType rt) { + int before = reg->type_count; + cbm_registry_add_type(reg, rt); + if (reg->type_count > before) { + c_type_first_note(ix, arena, reg, before); + } +} + void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin) { + if (result->lsp_skipped) + return; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -5695,6 +5959,18 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i const char *module_qn = result->module_qn; + /* Scratch for the registration index and the registry's hash index; both + * die with this call (quadratic-pattern #6: never per-file index memory in + * a longer-lived arena). */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + CTypeFirstIndex first_ix = {0}; + if (c_type_first_alloc(&first_ix, &idx_arena, 256)) { + for (int ti = 0; ti < reg.type_count; ti++) { + c_type_first_note(&first_ix, &idx_arena, ®, ti); + } + } + // Register file's own definitions for (int i = 0; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; @@ -5728,19 +6004,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i rt.embedded_types = embedded; } } - cbm_registry_add_type(®, rt); + c_add_type_indexed(®, &first_ix, &idx_arena, rt); } // Field definitions → populate field_names/field_types on parent type if (d->label && strcmp(d->label, "Field") == 0 && d->parent_class && d->return_type) { // Find or add the parent type in registry - CBMRegisteredType *parent_rt = NULL; - for (int ri = 0; ri < reg.type_count; ri++) { - if (strcmp(reg.types[ri].qualified_name, d->parent_class) == 0) { - parent_rt = ®.types[ri]; - break; - } - } + int parent_idx = c_type_first_find(&first_ix, ®, d->parent_class); + CBMRegisteredType *parent_rt = parent_idx >= 0 ? ®.types[parent_idx] : NULL; if (parent_rt) { // Count existing fields int existing = 0; @@ -5807,14 +6078,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i if (strcmp(d->label, "Method") == 0 && d->parent_class) { rf.receiver_type = d->parent_class; // Auto-create type if needed - if (!cbm_registry_lookup_type(®, d->parent_class)) { + if (c_type_first_find(&first_ix, ®, d->parent_class) < 0) { CBMRegisteredType auto_type; memset(&auto_type, 0, sizeof(auto_type)); auto_type.qualified_name = d->parent_class; // Extract short name const char *dot = strrchr(d->parent_class, '.'); auto_type.short_name = dot ? dot + 1 : d->parent_class; - cbm_registry_add_type(®, auto_type); + c_add_type_indexed(®, &first_ix, &idx_arena, auto_type); } } @@ -5822,13 +6093,27 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i } } + /* Hash-index the registry before the walk (#1527): unindexed, every + * resolver lookup scanned the stdlib plus all of this file's definitions, + * O(calls x defs) per file. First-registered chains keep each duplicated + * name resolving to the entry the scan returned; resolver additions after + * this point stay visible through the registry's tail scan. */ + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + // Initialize context and run CLSPContext ctx; c_lsp_init(&ctx, arena, source, source_len, ®, module_qn, cpp_mode, &result->resolved_calls); ctx.source_origin = source_origin; + c_memo_test_stage(&ctx, + source_origin == CBM_SOURCE_ORIGIN_PREPROCESSED ? "preprocessed" : "raw"); c_lsp_process_file(&ctx, root); - c_rewrite_proved_destructor_candidates(result); + if (ctx.eval_failure) + cbm_c_lsp_record_failure(result, ctx.eval_failure); + else + c_rewrite_proved_destructor_candidates(result); + cbm_arena_destroy(&idx_arena); } // ============================================================================ @@ -6000,13 +6285,22 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in return reg; } -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, cpp_mode, reg, include_paths, include_ns_qns, + include_count, cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, + int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -6014,14 +6308,14 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6031,20 +6325,32 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int for (int i = 0; i < include_count; i++) { c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_status(arena, source, source_len, module_qn, cpp_mode, defs, + def_count, include_paths, include_ns_qns, include_count, + cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out) - return; + return CBM_LSP_COMPLETE; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -6064,14 +6370,14 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6090,20 +6396,23 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } // --- Batch cross-file LSP --- -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchCLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -6119,10 +6428,12 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co memset(&file_out, 0, sizeof(file_out)); // Delegate to existing per-file function - cbm_run_c_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->cpp_mode, file->defs, file->def_count, file->include_paths, - file->include_ns_qns, file->include_count, file->cached_tree, - &file_out); + bool file_complete = cbm_run_c_lsp_cross( + &file_arena, file->source, file->source_len, file->module_qn, file->cpp_mode, + file->defs, file->def_count, file->include_paths, file->include_ns_qns, + file->include_count, file->cached_tree, &file_out); + + complete = complete && file_complete; // Copy results to output arena (must outlive per-file arena) if (file_out.count > 0) { @@ -6147,4 +6458,5 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co cbm_arena_destroy(&file_arena); } + return complete; } diff --git a/internal/cbm/lsp/c_lsp.h b/internal/cbm/lsp/c_lsp.h index af81d5bbdf..bbbf236555 100644 --- a/internal/cbm/lsp/c_lsp.h +++ b/internal/cbm/lsp/c_lsp.h @@ -99,12 +99,33 @@ typedef struct { // READ-ONLY across resolve workers — never mutate it // (and never store per-worker arena pointers into it) bool debug; - int eval_depth; // recursion depth for c_eval_expr_type (crash guard) - int eval_steps; // total expression eval calls for current file (hang guard) + int eval_depth; // recursion depth for c_eval_expr_type (crash guard) + uint32_t eval_truncations; // depth-cap cutoffs; a truncated result is never memoized + // Per-evaluation memo (#1527): results of the sub-expressions evaluated + // during ONE outermost c_eval_expr_type call, keyed by node identity. The + // evaluator re-evaluates a call's receiver (member lookup, then template + // substitution), which made an N-link fluent chain cost 2^N; the per-file + // step cap that used to hide that is gone. Evaluation never mutates scope + // or context, so within one outermost call a node's type cannot change; + // the memo is emptied when the next outermost call starts, because the + // resolver walk mutates scope between calls. + struct CLSPEvalMemoSlot *eval_memo; // open addressing, arena-owned + int *eval_memo_used; // slots filled by the current evaluation + int eval_memo_cap; // power of two, or 0 + int eval_memo_count; + CBMLSPStatus eval_failure; // sticky: stop refinement rather than repeat uncached work +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; // -1 = normal; 0 = fail the next memo allocation +#endif int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed } CLSPContext; +struct CLSPEvalMemoSlot { + const void *node_id; // TSNode.id; NULL = empty + const CBMType *result; +}; + // --- API --- void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int source_len, @@ -132,8 +153,28 @@ const CBMType *c_simplify_type(CLSPContext *ctx, const CBMType *t, bool unwrap_p void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin); +// A memo allocation/capacity failure is an explicit incomplete-refinement error. +#define CBM_C_LSP_DEPTH_ERROR "C/C++ expression recursion depth limit exceeded" +#define CBM_C_LSP_MEMO_ERROR "C/C++ expression memo allocation or capacity failure" +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +// Thread-local, scoped to memo allocations only; restore with -1 after a test. +void cbm_c_lsp_test_memo_fail_after(int successful_allocations); +void cbm_c_lsp_test_depth_fail(bool enabled); +uint32_t cbm_c_lsp_test_walks_take(void); +#endif + // Cross-file LSP: build registry from defs + stdlib, re-parse and resolve. -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +// Status variants distinguish memo/depth failure. Bool wrappers report success. +// Completed output before a failed walk is kept. +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, // NULL = parse internally @@ -146,7 +187,14 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in // Cross-file LSP using a pre-built shared registry (Tier 2). Skips the // per-file registry build; just parse + resolve. -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **include_paths, const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **include_paths, const char **include_ns_qns, @@ -178,7 +226,7 @@ typedef struct { // Process multiple C/C++ files' cross-file LSP in one CGo call. // out must point to file_count pre-zeroed CBMResolvedCallArray structs. -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out); #endif // CBM_LSP_C_LSP_H diff --git a/internal/cbm/lsp/kotlin_lsp.c b/internal/cbm/lsp/kotlin_lsp.c index 0ca49afd5a..c0e2f3febf 100644 --- a/internal/cbm/lsp/kotlin_lsp.c +++ b/internal/cbm/lsp/kotlin_lsp.c @@ -489,13 +489,8 @@ static const CBMScope *kt_scope_declaring_name(const CBMScope *scope, const char if (!name) return NULL; for (const CBMScope *current = scope; current; current = current->parent) { - for (const CBMScopeChunk *chunk = current->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return current; - } - } - } + if (cbm_scope_lookup_local(current, name)) + return current; } return NULL; } diff --git a/internal/cbm/lsp/lsp_node_iter.h b/internal/cbm/lsp/lsp_node_iter.h index 033962f14f..65db5c21b1 100644 --- a/internal/cbm/lsp/lsp_node_iter.h +++ b/internal/cbm/lsp/lsp_node_iter.h @@ -39,4 +39,33 @@ static inline TSNode* cbm_lsp_collect_children(CBMArena* arena, TSNode node, uin return kids; } +/* Named-children variant (#1527): the same single cursor pass, keeping only + * named nodes, so `for (i) ts_node_named_child(node, i)` over a wide node (a + * module holding thousands of classes, a body holding thousands of + * statements) becomes O(n). Returns NULL and sets *out_n=0 for a node without + * named children or on OOM; callers that must not lose children on OOM fall + * back to indexed access. */ +static inline TSNode *cbm_lsp_collect_named_children(CBMArena *arena, TSNode node, + uint32_t *out_n) { + uint32_t nc = ts_node_named_child_count(node); + *out_n = 0; + if (nc == 0) + return NULL; + TSNode *kids = (TSNode *)cbm_arena_alloc(arena, (size_t)nc * sizeof(TSNode)); + if (!kids) + return NULL; + uint32_t kn = 0; + TSTreeCursor cur = ts_tree_cursor_new(node); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode c = ts_tree_cursor_current_node(&cur); + if (ts_node_is_named(c)) + kids[kn++] = c; + } while (kn < nc && ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + *out_n = kn; + return kids; +} + #endif /* CBM_LSP_NODE_ITER_H */ diff --git a/internal/cbm/lsp/lsp_work.h b/internal/cbm/lsp/lsp_work.h new file mode 100644 index 0000000000..fa286293d3 --- /dev/null +++ b/internal/cbm/lsp/lsp_work.h @@ -0,0 +1,24 @@ +/* + * lsp_work.h — deterministic work counter for LSP complexity tests (#1527). + * + * Seam builds only (CBM_ENABLE_TEST_SEAMS): every name comparison a resolver + * primitive performs while searching (scope frame probes, enclosing-definition + * candidates) adds to a thread-local counter, so a test can assert that the + * work grows linearly with input size without reading a clock (O9). Product + * builds compile CBM_LSP_WORK to nothing. + */ +#ifndef CBM_LSP_WORK_H +#define CBM_LSP_WORK_H + +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +extern _Thread_local uint64_t cbm_lsp_work_steps; +#define CBM_LSP_WORK(n) (cbm_lsp_work_steps += (uint64_t)(n)) +/* Read and reset the calling thread's counter. */ +uint64_t cbm_lsp_work_take(void); +#else +#define CBM_LSP_WORK(n) ((void)0) +#endif + +#endif /* CBM_LSP_WORK_H */ diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 70de5dce78..e78bba7dcb 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -13,6 +13,7 @@ #include "py_lsp.h" #include "../cbm.h" #include "../helpers.h" +#include "lsp_node_iter.h" #include "tree_sitter/api.h" #include #include @@ -25,14 +26,80 @@ * only from py_lsp.c, never compiled standalone. */ #include "py_builtins.c" -/* Guards for py_eval_expr_type — mirrors c_eval_expr_type's guard design - * (C_EVAL_DEPTH_LIMIT / C_EVAL_MAX_STEPS_PER_FILE in c_lsp.c). */ +/* Stack guard for py_eval_expr_type (#720; same limit as C_EVAL_DEPTH_LIMIT). + * There is deliberately NO per-file work budget (#1527): the step cap that + * used to sit here stopped resolution after 10000 evaluations and left the + * rest of a large file silently unresolved while the file still reported + * status `indexed`. Memoized evaluation (#710), hashed scope frames, and + * cursor-based child access reduce the repeated work. Memo failure is reported + * explicitly; the existing depth bound now reports an incomplete walk too. */ #define PY_LSP_MAX_EVAL_DEPTH 256 -#define PY_EVAL_MAX_STEPS_PER_FILE 10000 +#include "lsp_work.h" + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int py_test_memo_allocations_left = -1; +static _Thread_local bool py_test_depth_failure; +void cbm_py_lsp_test_depth_fail(bool enabled) { + py_test_depth_failure = enabled; +} +void cbm_py_lsp_test_memo_fail_after(int successful_allocations) { + py_test_memo_allocations_left = successful_allocations; +} +#endif + +static void py_memo_test_stage(PyLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + const char *depth_stage = getenv("CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 256; + const char *fail_stage = getenv("CBM_TEST_PY_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime also works when the result arena cannot allocate. */ + result->error_msg = + status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR : CBM_PY_LSP_MEMO_ERROR; +} // Forward decls static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node); +/* Named children of one node for an index loop (#1527). ts_node_named_child(n, i) + * restarts tree-sitter's child iterator at the first child, so indexing every + * child of a wide node is O(n^2) -- a module of thousands of classes, or a body + * of thousands of statements, spent most of its resolve time re-walking its + * own children. Wide nodes are collected once (lsp_node_iter.h); narrow nodes + * keep plain indexed access, which is cheaper than an allocation. */ +enum { PY_KIDS_COLLECT_MIN = 64 }; +typedef struct { + TSNode node; + TSNode *kids; /* NULL = index the node directly */ + uint32_t n; +} PyKids; + +static PyKids py_kids(CBMArena *arena, TSNode node) { + PyKids k = {node, NULL, ts_node_named_child_count(node)}; + if (k.n >= PY_KIDS_COLLECT_MIN) { + uint32_t got = 0; + TSNode *kids = cbm_lsp_collect_named_children(arena, node, &got); + if (kids && got == k.n) + k.kids = kids; + } + return k; +} + +static TSNode py_kid(const PyKids *k, uint32_t i) { + return k->kids ? k->kids[i] : ts_node_named_child(k->node, i); +} + /* Decorators are extracted as raw syntax (`@property`, `@pkg.cache(...)`), * not resolved qualified names. Preserve the raw array for sound callable- * value admission, while parsing only enough of the spelling to retain the @@ -142,7 +209,7 @@ static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) { * The walk_depth-- runs after the inner returns, so early returns in the body * never leak the counter. */ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { - if (ctx->walk_depth >= cbm_lsp_max_walk_depth()) + if (!ctx || ctx->eval_failure || ctx->walk_depth >= cbm_lsp_max_walk_depth()) return; ctx->walk_depth++; py_resolve_calls_in_inner(ctx, node); @@ -229,6 +296,11 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou if (!ctx) return; memset(ctx, 0, sizeof(PyLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = py_test_memo_allocations_left; + if (py_test_depth_failure) + ctx->eval_depth = 256; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; @@ -505,16 +577,19 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod return py_import_match_result(&match, qn_io); } -static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local, - const char **qn_io) { +/* stmts: the root's import statements, collected once per root by the caller + * (#1527) -- only they can match, and rescanning every top-level statement + * once per import made classification O(imports x module size). */ +static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, bool have_root, + const TSNode *stmts, uint32_t stmt_count, + const char *local, const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; - if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root)) + if (!ctx || !local || !qn || !qn_io || !have_root) return PY_DIRECT_IMPORT_UNKNOWN; PyImportSyntaxMatch match = {0}; - uint32_t root_count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < root_count; i++) { - py_import_match_statement(ctx, ts_node_named_child(root, i), local, qn, &match); + for (uint32_t i = 0; i < stmt_count; i++) { + py_import_match_statement(ctx, stmts[i], local, qn, &match); } return py_import_match_result(&match, qn_io); } @@ -561,12 +636,32 @@ static void py_bind_dotted_prefixes(PyLSPContext *ctx, const char *qn) { static void py_classify_imports_for_root(PyLSPContext *ctx, TSNode root) { if (!ctx) return; + /* A null root classifies every import UNKNOWN, as it always has; so does + * a root whose statement list cannot be allocated (no proof either way). */ + bool have_root = !ts_node_is_null(root); + TSNode *stmts = NULL; + uint32_t stmt_count = 0; + if (have_root && ctx->import_count > 0) { + PyKids rk = py_kids(ctx->arena, root); + if (rk.n > 0) { + stmts = (TSNode *)cbm_arena_alloc(ctx->arena, (size_t)rk.n * sizeof(TSNode)); + if (!stmts) + have_root = false; + } + for (uint32_t i = 0; stmts && i < rk.n; i++) { + TSNode c = py_kid(&rk, i); + const char *ck = ts_node_type(c); + if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) + stmts[stmt_count++] = c; + } + } for (int i = 0; i < ctx->import_count; i++) { const char *local = ctx->import_local_names[i]; const char *qn = ctx->import_module_qns[i]; if (!local || !qn) continue; - PyDirectImportKind direct_kind = py_import_kind_from_ast(ctx, root, local, &qn); + PyDirectImportKind direct_kind = + py_import_kind_from_ast(ctx, have_root, stmts, stmt_count, local, &qn); ctx->import_module_qns[i] = qn; if (ctx->import_kinds) ctx->import_kinds[i] = (unsigned char)direct_kind; @@ -783,6 +878,8 @@ static const char *py_lookup_dict_dispatch(PyLSPContext *ctx, const char *var, c return NULL; } +enum { PY_RESOLVED_DEDUP_WINDOW = 256 }; + static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_qn, const char *strategy, float confidence, const char *reason, TSNode site) { @@ -796,9 +893,8 @@ static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_q // entries catches the common case while keeping per-emission O(1). // Without this cap the dedup is O(N) per emission -> O(N^2) per file // and dominates above ~1k call sites. - enum { DEDUP_WINDOW = 256 }; int n = ctx->resolved_calls->count; - int start = n > DEDUP_WINDOW ? n - DEDUP_WINDOW : 0; + int start = n > PY_RESOLVED_DEDUP_WINDOW ? n - PY_RESOLVED_DEDUP_WINDOW : 0; for (int i = start; i < n; i++) { CBMResolvedCall *rc = &ctx->resolved_calls->items[i]; if (rc->kind == CBM_RESOLVED_INVOCATION && rc->site_start_byte == site_start && @@ -1422,7 +1518,7 @@ static const CBMType *py_iterable_element_type(PyLSPContext *ctx, const CBMType } /* The real recursive-descent evaluator. Never call directly — go through - * the memoizing, depth- and budget-guarded py_eval_expr_type wrapper below + * the memoizing, depth-guarded py_eval_expr_type wrapper below * (every recursive call inside this body already does). */ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) @@ -2086,6 +2182,14 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { if (ctx->type_cache_cap >= PY_TYPE_CACHE_MAX_CAP) return false; int new_cap = ctx->type_cache_cap ? ctx->type_cache_cap * 2 : PY_TYPE_CACHE_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(CBMPyTypeCacheEntry)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return false; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif CBMPyTypeCacheEntry *new_entries = (CBMPyTypeCacheEntry *)cbm_arena_alloc( ctx->arena, (size_t)new_cap * sizeof(CBMPyTypeCacheEntry)); if (!new_entries) @@ -2111,9 +2215,9 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { return true; } -static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { +static bool py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { if (!id || !result) - return; + return true; /* Keep load strictly below 75% so probe chains stay short. If growth * fails (OOM / max cap), REJECT the insert rather than filling up: a * full table would turn every probe loop into a full-table scan, and @@ -2121,7 +2225,7 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp if (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3) { if (!py_type_cache_grow(ctx) && (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3)) - return; + return false; } uint32_t mask = (uint32_t)ctx->type_cache_cap - 1; uint32_t idx = py_type_cache_hash(id) & mask; @@ -2132,31 +2236,33 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp e->gen = ctx->type_cache_gen; e->result = result; ctx->type_cache_count++; - return; + return true; } if (e->node_id == id) { e->gen = ctx->type_cache_gen; /* refresh a stale-generation entry in place */ e->result = result; - return; + return true; } idx = (idx + 1) & mask; } /* Probe bound exhausted — only reachable with a corrupt count; drop the * insert instead of spinning. */ + return false; } -/* Memoizing, depth- and budget-guarded wrapper — the function every call - * site in this file goes through. */ +/* Memoizing, depth-guarded wrapper — the function every call site in this + * file goes through. */ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); /* Depth cap (issue #720): the evaluator recurses once per expression * nesting level, so a pathologically deep expression (tens of - * thousands of parens) overflowed the native stack. Same limit as - * C_EVAL_DEPTH_LIMIT. Past the cap: unknown, and NEVER cached. */ + * thousands of parens) overflowed the native stack. At the existing bound, + * stop this walk explicitly; never memoize a truncated result as complete. */ if (ctx->eval_depth >= PY_LSP_MAX_EVAL_DEPTH) { ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } @@ -2164,17 +2270,9 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { if (cached) return cached; - /* Per-file work budget (mirrors C_EVAL_MAX_STEPS_PER_FILE): expression - * type evaluation is best-effort, so pathological files degrade to - * unknown instead of stalling repository indexing. Only real - * evaluations consume budget — cache hits above are O(1). */ - if (ctx->eval_steps++ > PY_EVAL_MAX_STEPS_PER_FILE) { - ctx->eval_truncations++; - if (ctx->debug && ctx->eval_steps == PY_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [pylsp] expression eval step budget exhausted; returning unknown\n"); - } - return cbm_type_unknown(); - } + /* One real (non-memoized) evaluation: the work the complexity tests count + * to prove evaluation stays linear without a budget. */ + CBM_LSP_WORK(1); uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; @@ -2184,19 +2282,25 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { result = cbm_type_unknown(); /* Only cache full-fidelity results: if any descendant evaluation was - * cut off by the depth cap or the step budget, this result may be a - * truncated `unknown`. Caching it would poison later evaluations that - * reach the same node from a shallower frame (or with fresh budget) — - * exactly the reuse the guards are supposed to preserve. */ - if (ctx->eval_truncations == trunc_before) - py_type_cache_insert(ctx, node.id, result); - return result; + * cut off by the depth cap, this result may be a truncated `unknown`. + * The sticky failure stops this walk instead of repeating uncached branches; + * a fresh walk starts with a fresh cache. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !py_type_cache_insert(ctx, node.id, result)) + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + return ctx->eval_failure ? cbm_type_unknown() : result; } +#ifdef CBM_ENABLE_TEST_SEAMS +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node) { + return py_eval_expr_type(ctx, node); +} +#endif + /* ── statement processing: bind from assignments, for-loops, with-as ──── */ static void py_process_statement(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return; const char *k = ts_node_type(node); @@ -2502,6 +2606,8 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { /* ── Recursive walker: process statements + emit resolved_calls ── */ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { + if (!ctx || ctx->eval_failure) + return; if (!ctx || ctx->callable_value_proof_disabled) return; TSNode fn = ts_node_child_by_field_name(call_node, "function", 8); @@ -3443,9 +3549,9 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { return; } - uint32_t nc = ts_node_named_child_count(node); - for (uint32_t i = 0; i < nc; i++) { - py_resolve_calls_in(ctx, ts_node_named_child(node, i)); + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n; i++) { + py_resolve_calls_in(ctx, py_kid(&nk, i)); } } @@ -4006,6 +4112,8 @@ static void py_bind_parameters(PyLSPContext *ctx, TSNode params) { } static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char *container_qn) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(func_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4034,9 +4142,9 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char if (!ts_node_is_null(body)) { py_resolve_calls_in(ctx, body); // Also descend into nested function/class definitions in the body. - uint32_t bnc = ts_node_named_child_count(body); - for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + PyKids bk = py_kids(ctx->arena, body); + for (uint32_t i = 0; i < bk.n; i++) { + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, ctx->enclosing_func_qn); @@ -4064,6 +4172,8 @@ static bool py_is_init_method(PyLSPContext *ctx, TSNode func_node) { } static void py_process_class(PyLSPContext *ctx, TSNode class_node) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(class_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4076,13 +4186,14 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { TSNode body = ts_node_child_by_field_name(class_node, "body", 4); if (!ts_node_is_null(body)) { - uint32_t bnc = ts_node_named_child_count(body); + PyKids bk = py_kids(ctx->arena, body); + uint32_t bnc = bk.n; // First pass: process class-level annotated assignments (PEP 526 // class-body field annotations like `x: int`) and dunder __init__ // methods so fields are registered before sibling methods that // reference them. for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "expression_statement") == 0 && ts_node_named_child_count(c) > 0) { TSNode inner = ts_node_named_child(c, 0); @@ -4105,7 +4216,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Second pass: __init__ (so self.x assignments populate fields). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); TSNode fn_node = c; bool is_decorated = strcmp(ck, "decorated_definition") == 0; @@ -4123,7 +4234,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Third pass: every other method (and nested classes). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { if (!py_is_init_method(ctx, c)) { @@ -4148,10 +4259,17 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { /* Module bindings are replayed in source order before deferred function bodies * are analyzed. Class definitions always replace an earlier callable identity; * the registry decides whether the resulting class type is known precisely. */ -static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const char *name) { - uint32_t count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < count; i++) { - TSNode node = ts_node_named_child(root, i); +/* The names of the classes this source defines at module level, as a name set + * (a detached, hash-indexed CBMScope frame). Built in one pass over the root: + * asking "does the root define a class named X?" by rescanning the root once + * per registry type made module assembly O(types x top-level statements). + * Returns NULL if a class name or set entry cannot be collected completely. */ +static CBMScope *py_root_class_names(PyLSPContext *ctx, const PyKids *rk) { + CBMScope *names = cbm_scope_push(ctx->arena, NULL); + if (!names) + return NULL; + for (uint32_t i = 0; i < rk->n; i++) { + TSNode node = py_kid(rk, i); const char *kind = ts_node_type(node); if (strcmp(kind, "decorated_definition") == 0) { node = ts_node_child_by_field_name(node, "definition", 10); @@ -4159,20 +4277,33 @@ static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const ch } if (strcmp(kind, "class_definition") != 0) continue; - TSNode name_node = ts_node_child_by_field_name(node, "name", 4); - if (py_import_node_text_equals(ctx, name_node, name)) - return true; + char *name = py_node_text(ctx, ts_node_child_by_field_name(node, "name", 4)); +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_root_name_failure == 1) + name = NULL; + else if (ctx->test_root_name_failure == 2) + name = ""; +#endif + if (!name || !name[0] || !cbm_scope_bind_checked(names, name, cbm_type_unknown())) + return NULL; } - return false; + return names; } /* A project registry can contain another file's class in the same logical * Python module. Keep those cross-file globals available, but do not prebind * classes declared by this source: their binding epoch belongs in the ordered * replay below. */ -static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { +static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) { if (!ctx || !ctx->registry || !ctx->module_qn) return; + CBMScope *defined = py_root_class_names(ctx, rk); + if (!defined) { + /* Without the set no external class can be proven undeclared here; + * binding none is the fail-closed direction. */ + py_disable_callable_value_proof(ctx); + return; + } size_t prefix_len = strlen(ctx->module_qn); CBMTypeShortIter all_types; cbm_registry_all_types_chain(ctx->registry, &all_types); @@ -4181,13 +4312,20 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { const char *qn = type->qualified_name; const char *name = type->short_name; if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || - py_root_defines_class_named(ctx, root, name)) { + cbm_scope_lookup_local(defined, name)) { continue; } py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); } } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root) { + PyKids rk = py_kids(ctx->arena, root); + py_bind_external_module_classes(ctx, &rk); +} +#endif + static void py_bind_module_class(PyLSPContext *ctx, TSNode class_node) { if (!ctx || ts_node_is_null(class_node) || !ctx->module_qn) return; @@ -4564,9 +4702,9 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int } return; } - uint32_t count = ts_node_named_child_count(node); - for (uint32_t i = 0; i < count; i++) - py_invalidate_possible_bindings(ctx, ts_node_named_child(node, i), depth + 1); + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n; i++) + py_invalidate_possible_bindings(ctx, py_kid(&nk, i), depth + 1); } static bool py_replayable_import_kind(PyDirectImportKind kind) { @@ -4654,10 +4792,22 @@ static bool py_expression_is_annotation_only_assignment(TSNode statement) { } void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { - if (!ctx || ts_node_is_null(root)) + if (!ctx || ctx->eval_failure || ts_node_is_null(root)) return; + int resolved_start = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + int synthetic_start = ctx->syn_calls ? ctx->syn_calls->count : 0; + /* Only invocation dedup mutates prior rows, within this fixed tail. + * Snapshot once per walk: no full-output copy or fallible rollback allocation. */ + int tail_count = + resolved_start < PY_RESOLVED_DEDUP_WINDOW ? resolved_start : PY_RESOLVED_DEDUP_WINDOW; + int tail_start = resolved_start - tail_count; + CBMResolvedCall saved_tail[PY_RESOLVED_DEDUP_WINDOW]; + CBMResolvedCall *original_tail = tail_count ? ctx->resolved_calls->items + tail_start : NULL; + if (tail_count) + memcpy(saved_tail, original_tail, (size_t)tail_count * sizeof(*saved_tail)); + PyKids rk = py_kids(ctx->arena, root); py_classify_imports_for_root(ctx, root); - py_bind_external_module_classes(ctx, root); + py_bind_external_module_classes(ctx, &rk); unsigned char *consumed_imports = NULL; if (ctx->import_count > 0) { consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); @@ -4665,13 +4815,13 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { memset(consumed_imports, 0, (size_t)ctx->import_count); } - uint32_t nc = ts_node_named_child_count(root); + uint32_t nc = rk.n; const char *prev_func = ctx->enclosing_func_qn; ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.__module__", ctx->module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) { py_replay_import_statement(ctx, c, consumed_imports); @@ -4709,8 +4859,8 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } // Pass 2: top-level calls (rare) and nested definitions. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, NULL); @@ -4729,6 +4879,17 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } ctx->enclosing_func_qn = prev_func; + if (ctx->eval_failure) { + if (tail_count) { + memcpy(original_tail, saved_tail, (size_t)tail_count * sizeof(*saved_tail)); + memcpy(ctx->resolved_calls->items + tail_start, saved_tail, + (size_t)tail_count * sizeof(*saved_tail)); + } + if (ctx->resolved_calls) + ctx->resolved_calls->count = resolved_start; + if (ctx->syn_calls) + ctx->syn_calls->count = synthetic_start; + } } /* Register one definition into the registry. Returns true if recognized. */ @@ -4887,7 +5048,7 @@ static bool py_register_def(CBMArena *arena, CBMTypeRegistry *reg, CBMDefinition void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root) { - if (!arena || !result) + if (!arena || !result || result->lsp_skipped) return; /* Inject minimal builtin definitions as real graph nodes (builtins.len, @@ -4913,6 +5074,18 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, } py_mark_ambiguous_callable_bindings(®); + /* Hash-index the registry before the walk (#1527). Unfinalized, every + * lookup scanned the stdlib plus all of this file's definitions, which + * made resolving a large module O(calls x defs). The index answers each + * duplicated name with its first registration, as the scan did, so the + * resolution result is unchanged; post-walk additions (instance fields) + * stay visible through the registry's tail scan. The index lives in a + * scratch arena that dies with this call, not in the result arena. */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + PyLSPContext ctx; py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, &result->resolved_calls); /* Let the resolver inject synthetic syntactic calls for operator/subscript @@ -4926,7 +5099,11 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, } } + py_memo_test_stage(&ctx, "raw"); py_lsp_process_file(&ctx, root); + if (ctx.eval_failure) + cbm_py_lsp_record_failure(result, ctx.eval_failure); + cbm_arena_destroy(&idx_arena); } /* ── Cross-file + batch ───────────────────────────────────────── */ @@ -5094,13 +5271,23 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe } } -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_status(arena, source, source_len, module_qn, defs, def_count, + import_names, import_qns, import_count, cached_tree, out, + synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5108,13 +5295,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); @@ -5141,6 +5328,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); cbm_arena_destroy(&idx_arena); @@ -5148,6 +5336,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } /* ── Tier 2: pre-built per-language registry (mirrors Go pilot) ── */ @@ -5175,13 +5364,22 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i return reg; } -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, reg, import_names, import_qns, import_count, + cached_tree, out, synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5189,13 +5387,13 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); @@ -5208,19 +5406,22 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); if (owns_tree && tree) ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!arena || !files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchPyLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -5233,9 +5434,10 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ CBMResolvedCallArray file_out; memset(&file_out, 0, sizeof(file_out)); - cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->defs, file->def_count, file->import_names, file->import_qns, - file->import_count, file->cached_tree, &file_out, NULL); + if (!cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, + file->defs, file->def_count, file->import_names, file->import_qns, + file->import_count, file->cached_tree, &file_out, NULL)) + complete = false; if (file_out.count > 0) { out[f].count = file_out.count; @@ -5264,6 +5466,7 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ cbm_arena_destroy(&file_arena); } + return complete; } /* ── Stdlib stub — Phase 10 replaces with auto-generated body ─── */ diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 939025100f..e7536b8f9e 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -95,14 +95,18 @@ typedef struct { // py_eval_expr_type memoization + guards (issues #710/#720; mirrors // c_eval_expr_type's guard design in c_lsp.c). All zero via memset — - // each file starts with a cold cache and a full budget. + // each file starts with a cold cache. CBMPyTypeCacheEntry *type_cache; // open addressing, linear probe, arena-allocated int type_cache_count; // occupied slots (kept < 75% of cap) int type_cache_cap; // power-of-two capacity uint32_t type_cache_gen; // bumped on every scope mutation (O(1) flush) int eval_depth; // evaluator recursion depth (PY_LSP_MAX_EVAL_DEPTH) - int eval_steps; // per-file work budget used (PY_EVAL_MAX_STEPS_PER_FILE) - uint32_t eval_truncations; // depth/budget cutoff count — gates memo inserts + uint32_t eval_truncations; // depth-cap cutoff count — gates memo inserts + CBMLSPStatus eval_failure; // sticky: memo/depth failure makes this walk incomplete +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; + int test_root_name_failure; // 1 = NULL, 2 = empty; only the root-name collection +#endif // Sticky fail-closed guard for exact callable-value proof. If a required // scope/name/overlay allocation fails, later semantic passes must leave @@ -161,9 +165,28 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, // Phase 10 fills the body; Phase 2 ships a no-op so linkage works. void cbm_python_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); +#define CBM_PY_LSP_DEPTH_ERROR "Python expression recursion depth limit exceeded" +#define CBM_PY_LSP_MEMO_ERROR "Python expression memo allocation or capacity failure" +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_memo_fail_after(int successful_allocations); +void cbm_py_lsp_test_depth_fail(bool enabled); +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root); +#endif + // --- Cross-file LSP resolution --- +// Status variants distinguish memo/depth failure; bool wrappers report success. +// Existing no-op/parser exits retain their previous behavior. + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, // NULL = parse internally @@ -173,7 +196,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, * Filters all_defs to Python entries, builds + finalizes once. */ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count); -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); + +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **import_names, const char **import_qns, @@ -194,7 +223,7 @@ typedef struct { int import_count; } CBMBatchPyLSPFile; -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out); #endif // CBM_LSP_PY_LSP_H diff --git a/internal/cbm/lsp/scope.c b/internal/cbm/lsp/scope.c index 2d9740db0c..0b3c6505cb 100644 --- a/internal/cbm/lsp/scope.c +++ b/internal/cbm/lsp/scope.c @@ -1,5 +1,21 @@ #include "scope.h" +#include "lsp_work.h" #include +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +_Thread_local uint64_t cbm_lsp_work_steps = 0; +uint64_t cbm_lsp_work_take(void) { + uint64_t v = cbm_lsp_work_steps; + cbm_lsp_work_steps = 0; + return v; +} +#endif + +enum { + SCOPE_INDEX_MIN_CAP = 64, /* first table: > 2x one chunk, a power of two */ + SCOPE_INDEX_GROW = 2, +}; CBMScope* cbm_scope_push(CBMArena* a, CBMScope* current) { CBMScope* scope = (CBMScope*)cbm_arena_alloc(a, sizeof(CBMScope)); @@ -33,6 +49,114 @@ static CBMScopeChunk* alloc_chunk(CBMScope* scope) { return c; } +static uint32_t scope_name_hash(const char *name) { + uint32_t h = 2166136261u; /* FNV-1a */ + for (const unsigned char *p = (const unsigned char *)name; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Place one binding into a table that has room (load kept below 1/2). */ +static void scope_index_place(CBMVarBinding **slots, int cap, CBMVarBinding *b) { + uint32_t mask = (uint32_t)cap - 1; + uint32_t i = scope_name_hash(b->name) & mask; + while (slots[i]) { + i = (i + 1) & mask; + } + slots[i] = b; +} + +/* (Re)build the frame's table at cap slots from its chunks. On allocation + * failure the frame drops its index and stays on the always-correct scan. */ +static void scope_index_rebuild(CBMScope *scope, int cap) { + /* Recovery can start with many bindings and no index. Always size from + * the complete frame, never just from the previous table's capacity. */ + if (scope->binding_count < 0 || scope->binding_count > INT_MAX / 2) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + if (cap < SCOPE_INDEX_MIN_CAP) + cap = SCOPE_INDEX_MIN_CAP; + while (cap < scope->binding_count * 2) { + if (cap > INT_MAX / SCOPE_INDEX_GROW) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + cap *= SCOPE_INDEX_GROW; + } + if ((size_t)cap > SIZE_MAX / sizeof(CBMVarBinding *)) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + CBMVarBinding **slots = + scope->arena + ? (CBMVarBinding **)cbm_arena_alloc(scope->arena, (size_t)cap * sizeof(CBMVarBinding *)) + : NULL; + if (!slots) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + memset(slots, 0, (size_t)cap * sizeof(CBMVarBinding *)); + int count = 0; + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + if (c->bindings[i].name) { + scope_index_place(slots, cap, &c->bindings[i]); + count++; + } + } + } + scope->index.slots = slots; + scope->index.cap = cap; + scope->index.count = count; +} + +/* Record a freshly appended binding in the frame's index, building the index + * the first time the frame outgrows one chunk. */ +static void scope_index_note(CBMScope *scope, CBMVarBinding *b) { + if (scope->index.cap == 0) { + if (scope->binding_count > CBM_SCOPE_CHUNK_BINDINGS) { + scope_index_rebuild(scope, SCOPE_INDEX_MIN_CAP); + } + return; + } + if (scope->index.count >= scope->index.cap / 2) { + scope_index_rebuild(scope, scope->index.cap); /* includes b; sizes from binding_count */ + return; + } + scope_index_place(scope->index.slots, scope->index.cap, b); + scope->index.count++; +} + +/* The binding of name in one frame: hashed when the frame has an index, + * otherwise the linear chunk scan (small frames, or no index memory). */ +static CBMVarBinding *scope_frame_find(const CBMScope *scope, const char *name) { + if (scope->index.cap > 0) { + uint32_t mask = (uint32_t)scope->index.cap - 1; + for (uint32_t i = scope_name_hash(name) & mask;; i = (i + 1) & mask) { + CBMVarBinding *b = scope->index.slots[i]; + if (!b) { + return NULL; + } + CBM_LSP_WORK(1); + if (strcmp(b->name, name) == 0) { + return b; + } + } + } + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + CBM_LSP_WORK(1); + if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { + return &c->bindings[i]; + } + } + } + return NULL; +} + /* Returns false when the binding could NOT be recorded in THIS frame. * * The failure that matters is arena exhaustion in alloc_chunk: the old void @@ -47,14 +171,11 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp if (!scope || !name) { return false; } - for (CBMScopeChunk* c = scope->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].type = type; - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *existing = scope_frame_find(scope, name); + if (existing) { + existing->type = type; + existing->callable_qn = callable_qn; + return true; } CBMScopeChunk* head = scope->chunks; if (!head || head->used >= CBM_SCOPE_CHUNK_BINDINGS) { @@ -63,10 +184,13 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp return false; /* arena exhausted: the shadow did NOT take effect */ } } - head->bindings[head->used].name = name; - head->bindings[head->used].type = type; - head->bindings[head->used].callable_qn = callable_qn; + CBMVarBinding *b = &head->bindings[head->used]; + b->name = name; + b->type = type; + b->callable_qn = callable_qn; head->used++; + scope->binding_count++; + scope_index_note(scope, b); return true; } @@ -88,6 +212,13 @@ bool cbm_scope_bind_callable_checked(CBMScope *scope, const char *name, const CB return cbm_scope_bind_value(scope, name, type, callable_qn); } +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name) { + if (!scope || !name) { + return NULL; + } + return scope_frame_find(scope, name); +} + const CBMType* cbm_scope_lookup(const CBMScope* scope, const char* name) { const CBMVarBinding *binding = cbm_scope_lookup_binding(scope, name); if (binding) @@ -109,13 +240,10 @@ bool cbm_scope_update_callable(CBMScope *scope, const char *name, const char *ca return false; } for (CBMScope *s = scope; s != NULL; s = s->parent) { - for (CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *b = scope_frame_find(s, name); + if (b) { + b->callable_qn = callable_qn; + return true; } } return false; diff --git a/internal/cbm/lsp/scope.h b/internal/cbm/lsp/scope.h index 5ba7265a36..af14cb7cd2 100644 --- a/internal/cbm/lsp/scope.h +++ b/internal/cbm/lsp/scope.h @@ -25,27 +25,48 @@ typedef struct CBMScopeChunk { struct CBMScopeChunk* next; } CBMScopeChunk; +/* Name index for one frame (#1527). A frame that outgrows one chunk (a module + * scope binding thousands of classes and functions) is looked up through an + * open-addressing table of binding pointers instead of a linear chunk scan -- + * the scan made every identifier lookup O(frame size) and a whole file + * O(n^2). Bindings never move (chunks are arena-allocated and only + * prepended) and a name is bound at most once per frame (rebinding updates + * in place), so the table maps each name to exactly the binding the linear + * scan would have found. The table is an accelerator only: when it cannot be + * allocated or grown the frame drops it and every lookup falls back to the + * scan, which is always correct. */ +typedef struct { + CBMVarBinding **slots; // arena-owned; NULL = empty + int cap; // power of two, or 0 = no index (linear scan) + int count; +} CBMScopeIndex; + typedef struct CBMScope { struct CBMScope* parent; CBMScopeChunk* chunks; CBMArena* arena; // owning arena, propagated to children at push time + int binding_count; // bindings recorded in THIS frame + CBMScopeIndex index; // built once binding_count exceeds one chunk } CBMScope; +/* The binding of name in THIS frame only (no parent walk), or NULL. */ +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name); + /* Return the complete nearest binding in one scope-chain walk, or NULL when * unbound. Keep this internal hot-path primitive inline: all language * resolvers use cbm_scope_lookup(), while Python also consumes the complete - * record to avoid repeating the same linear scan. */ + * record to avoid repeating the same walk. Each frame is probed through + * cbm_scope_lookup_local(), which uses the frame's hashed name index once the + * frame outgrows one chunk (#1527), so a lookup costs one probe per frame + * rather than a scan of every binding in scope. */ static inline const CBMVarBinding *cbm_scope_lookup_binding(const CBMScope *scope, const char *name) { if (!name) return NULL; for (const CBMScope *s = scope; s != NULL; s = s->parent) { - for (const CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) - return &c->bindings[i]; - } - } + const CBMVarBinding *b = cbm_scope_lookup_local(s, name); + if (b) + return b; } return NULL; } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 7fe5055fd2..a78c7433db 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -1391,12 +1391,8 @@ static const CBMScope *ts_scope_binding_owner(const CBMScope *scope, const char return NULL; } for (const CBMScope *s = scope; s; s = s->parent) { - for (const CBMScopeChunk *chunk = s->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return s; - } - } + if (cbm_scope_lookup_local(s, name)) { + return s; } } return NULL; diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 06b55c391d..216c347794 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -1,4 +1,5 @@ #include "type_registry.h" +#include "lsp_work.h" #include #include @@ -56,7 +57,12 @@ static void build_qn_index(CBMTypeRegistry *reg, CBMArena *idx_arena, bool for_f for (int i = 0; i < bucket_count; i++) buckets[i] = -1; - for (int i = 0; i < count; i++) { + /* Chains are built by prepending, so the entry inserted LAST is found + * first. Insert descending when the registry asked for first-registered + * answers (see index_first_registered). */ + bool first = reg->index_first_registered; + for (int k = 0; k < count; k++) { + int i = first ? count - 1 - k : k; const char *qn = for_funcs ? reg->funcs[i].qualified_name : reg->types[i].qualified_name; if (!qn) continue; @@ -96,7 +102,9 @@ static void build_method_index(CBMTypeRegistry *reg, CBMArena *idx_arena) { buckets[i] = -1; int idx = 0; - for (int i = 0; i < reg->func_count; i++) { + bool first = reg->index_first_registered; /* see build_qn_index */ + for (int k = 0; k < reg->func_count; k++) { + int i = first ? reg->func_count - 1 - k : k; const CBMRegisteredFunc *f = ®->funcs[i]; if (!f->receiver_type || !f->short_name) continue; @@ -779,6 +787,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->method_bucket_count - 1)); for (int idx = reg->method_buckets[slot]; idx >= 0; idx = reg->method_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->method_entries[idx].hash != h) continue; const CBMRegisteredFunc *f = ®->funcs[reg->method_entries[idx].payload_index]; @@ -792,6 +801,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, * post-finalize additions stay visible instead of silently vanishing. */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -802,6 +812,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, for (int i = 0; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -829,6 +840,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->type_qn_bucket_count - 1)); for (int idx = reg->type_qn_buckets[slot]; idx >= 0; idx = reg->type_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->type_qn_entries[idx].hash != h) continue; int p = reg->type_qn_entries[idx].payload_index; @@ -839,6 +851,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } /* Tail-scan types added after finalize (see lookup_method_self). */ for (int i = reg->type_qn_entry_count; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (reg->types[i].qualified_name && strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; @@ -848,6 +861,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; } @@ -873,6 +887,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->func_qn_bucket_count - 1)); for (int idx = reg->func_qn_buckets[slot]; idx >= 0; idx = reg->func_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->func_qn_entries[idx].hash != h) continue; int p = reg->func_qn_entries[idx].payload_index; @@ -883,6 +898,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } /* Tail-scan funcs added after finalize (see lookup_method_self). */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (reg->funcs[i].qualified_name && strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; @@ -892,6 +908,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; } diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index c752350b17..1ee284bf7c 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -143,6 +143,13 @@ typedef struct CBMTypeRegistry { * every lookup linear-scans it -> O(files*defs) (the Linux-kernel full-index * hang) plus a heap data race across workers. */ bool read_only; + // Set before finalize by a caller whose lookups used to run unfinalized + // (#1527): the exact-QN and method indexes then chain entries in + // registration order, so a duplicated name resolves to its FIRST + // registration -- the answer the linear scan gave. Without it the + // hashed chains answer with the LAST registration (the Tier-2 behavior, + // left unchanged). + bool index_first_registered; } CBMTypeRegistry; // Initialize a registry. diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 66751a6829..6629956e7e 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -1431,14 +1431,22 @@ void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source, int * imports — the existing pipeline doesn't carry C-style include * resolution as a separate map, so pass NULL/0 and let the LSP * fall back to its own #include scan. */ - cbm_run_c_lsp_cross(&scratch, source, source_len, module_qn, cpp_mode, defs, def_count, - NULL, NULL, 0, tree, &out); + { + CBMLSPStatus status = + cbm_run_c_lsp_cross_status(&scratch, source, source_len, module_qn, cpp_mode, defs, + def_count, NULL, NULL, 0, tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(r, status); + } break; } - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out, &synthetic_calls); - break; + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_status(&scratch, source, source_len, module_qn, + defs, def_count, imp_names, imp_qns, + imp_count, tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(r, status); + } break; case CBM_LANG_PHP: cbm_run_php_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out); @@ -1586,18 +1594,24 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * &out); used_prebuilt = true; break; - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, - imp_keys, imp_vals, imp_count, result->cached_tree, - &out, &synthetic_calls); + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, &overlay, imp_keys, imp_vals, imp_count, + result->cached_tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_C: case CBM_LANG_CPP: - case CBM_LANG_CUDA: - cbm_run_c_lsp_cross_with_registry(&scratch, source, source_len, def_module, - (lang != CBM_LANG_C), &overlay, imp_keys, imp_vals, - imp_count, result->cached_tree, &out); + case CBM_LANG_CUDA: { + CBMLSPStatus status = cbm_run_c_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, (lang != CBM_LANG_C), &overlay, imp_keys, + imp_vals, imp_count, result->cached_tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_CSHARP: @@ -1860,10 +1874,14 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * * file, not to a stale extraction marker (the innocent-quarantine * failure mode). */ cbm_index_mark_start(files[i].rel_path); + bool had_error = cache[i]->has_error; cbm_pxc_dispatch_file(lang, cache[i], source, source_len, files[i].rel_path, def_modules[i], &cross_registries, module_def_index, all_defs, def_count, imp_keys, imp_vals, imp_count, NULL, NULL); cbm_index_mark_done(files[i].rel_path); + if (!had_error && cache[i]->has_error && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, cache[i]->error_msg, + "lsp_skipped"); per_lang_calls++; processed++; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a91acaf831..726199fde8 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -75,6 +75,8 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" #include "lsp/rust_cargo.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ #include "pipeline/worker_pool.h" #include "foundation/compat.h" @@ -1292,12 +1294,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) { snprintf(how_far, sizeof(how_far), "%u/%u nodes walked", result->walk_nodes_visited, result->tree_nodes); pp_err_add(errs, fi->rel_path, how_far, "walk_truncated"); - } else if (result->lsp_skipped) { - /* Indexed, but without the per-file and cross-file LSP refinement: - * the same kind of hole from the other direction. Nothing in - * production sets this any more except a truncated walk (handled - * above) and the test seam — it is reported anyway, so that if - * something sets it again the gap arrives named, not silent. */ + } else if (result->lsp_skipped && !result->has_error) { + /* has_error already recorded its precise cause above. Indexed, but without the per-file + * and cross-file LSP refinement: the same kind of hole from the other direction. + * Nothing in production sets this any more except a truncated walk (handled above) and + * the test seam — it is reported anyway, so that if something sets it again the gap + * arrives named, not silent. */ char size_text[CBM_SZ_64]; snprintf(size_text, sizeof(size_text), "%u nodes", result->tree_nodes); pp_err_add(errs, fi->rel_path, size_text, "lsp_skipped"); @@ -1916,6 +1918,9 @@ typedef struct { /* Counters for parallel.resolve.lsp_cross_done summary. */ _Atomic int lsp_cross_processed; _Atomic int lsp_cross_skipped_no_source; + /* Each file is owned by one worker. The coordinator reports failures + * after join, without mutating the pipeline's error list concurrently. */ + const char **lsp_failures; /* Per-sub-phase timing (ns aggregated across workers) — surfaces * exactly where parallel_resolve's wall time is spent so we stop @@ -3748,11 +3753,20 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * file around the resolve so a hang HERE is attributed to * this file, not to a stale extraction marker. */ cbm_index_mark_start(rel); + bool had_error = result->has_error; cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, rc->cross_registries, rc->module_def_index, rc->all_defs, rc->def_count, imp_keys, imp_vals, imp_count, pp_rust_shared_registry_get, rc); cbm_index_mark_done(rel); + if (!had_error && result->has_error && result->error_msg && + (strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0)) { + rc->lsp_failures[file_idx] = result->error_msg; + ws->errors++; + } /* Free the on-demand re-read (no-op when source was retained). */ free_source(lsp_source_owned); /* Contract: cbm_slab_reclaim() requires the thread parser to be @@ -3865,6 +3879,11 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return CBM_NOT_FOUND; } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); + const char **lsp_failures = calloc((size_t)file_count, sizeof(*lsp_failures)); + if (!lsp_failures) { + cbm_aligned_free(workers); + return CBM_NOT_FOUND; + } bool have_rust = false; for (int i = 0; i < file_count; i++) { @@ -3905,6 +3924,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .module_def_index = module_def_index, .cross_registries = cross_registries, .rust_manifest = rust_manifest_ptr, + .lsp_failures = lsp_failures, }; atomic_init(&rc.next_file_idx, 0); atomic_init(&rc.lsp_cross_processed, 0); @@ -3922,6 +3942,12 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_scale_end(&rc.scale); CBM_PROF_END_N("parallel_resolve", "1_dispatch_workers_parallel", t_resolve_dispatch, file_count); + for (int i = 0; i < file_count; i++) { + if (lsp_failures[i] && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], + "lsp_skipped"); + } + free(lsp_failures); /* Workers joined: the shared Rust registry (if built) is no longer read. * Free its dedicated arena + the lock (registry was self-contained: it strdup'd * all QNs, so freeing all_defs afterward is safe). */ diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index d890911506..dbbef1d850 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -19,6 +19,7 @@ * - DLL patterns, SFINAE, placement new */ #include "test_framework.h" +#include "../src/foundation/compat.h" #include "cbm.h" #include "lsp/c_lsp.h" #include "lsp/py_lsp.h" @@ -26,7 +27,9 @@ #include "lsp/ts_lsp.h" #include "lsp/go_lsp.h" #include "lsp/type_registry.h" +#include "lsp/lsp_work.h" #include "../src/pipeline/lsp_resolve.h" +#include "../src/pipeline/pass_lsp_cross.h" #include "arena.h" #include "preprocessor.h" #include @@ -16499,7 +16502,479 @@ TEST(clsp_preprocessed_destructor_rewrite_respects_origin_during_rewrite) { PASS(); } +/* ── #1527: no per-file eval budget; cost fixed at the root ────────────── */ + +/* Resolved rows that name a real target (not an lsp_unresolved marker) whose + * callee QN contains sub. */ +static int count_real_resolved(const CBMFileResult *r, const char *sub) { + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + if (rc->callee_qn && strstr(rc->callee_qn, sub) && rc->confidence > 0.0f && + !(rc->strategy && strcmp(rc->strategy, "lsp_unresolved") == 0)) + n++; + } + return n; +} + +/* A versionbits-style fluent chain (the #323 hang): every link returns + * Tester&, and the evaluator re-evaluates a call's receiver for template + * substitution, so without a memo an N-link chain costs 2^N evaluations. The + * per-file step cap that used to contain it is gone; the per-evaluation memo + * must keep the work polynomial AND every link must resolve. */ +static char *clsp_chain_source(int links, int stmts) { + size_t sz = 512 + (size_t)stmts * (32 + (size_t)links * 40); + char *src = malloc(sz); + if (!src) + return NULL; + size_t pos = (size_t)snprintf(src, sz, + "struct Tester {\n" + " Tester& Mine(int h, int t, int v);\n" + " Tester& TestDefined();\n" + " Tester& TestStateSinceHeight(int h);\n" + " Tester& Reset();\n" + "};\n" + "void run() {\n" + " Tester t;\n"); + for (int s = 0; s < stmts; s++) { + pos += (size_t)snprintf(src + pos, sz - pos, " t.Reset()"); + for (int l = 0; l < links; l++) { + if (l % 3 == 0) + pos += (size_t)snprintf(src + pos, sz - pos, ".Mine(%d, %d, 0x100)", l, l); + else if (l % 3 == 1) + pos += (size_t)snprintf(src + pos, sz - pos, ".TestDefined()"); + else + pos += (size_t)snprintf(src + pos, sz - pos, ".TestStateSinceHeight(%d)", l); + } + pos += (size_t)snprintf(src + pos, sz - pos, ";\n"); + } + snprintf(src + pos, sz - pos, "}\n"); + return src; +} + +TEST(clsp_issue1527_fluent_chain_work_is_polynomial) { + enum { LINKS = 14, STMTS = 3 }; + char *src = clsp_chain_source(LINKS, STMTS); + ASSERT_NOT_NULL(src); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + ASSERT_NOT_NULL(r); + printf(" chain %dx%d: work=%llu\n", LINKS, STMTS, (unsigned long long)work); + /* Every link of every statement resolves to its Tester method. */ + ASSERT_EQ(count_real_resolved(r, ".Tester."), STMTS * (LINKS + 1)); + /* Memoized: ~LINKS^2 evaluations per statement (each call node starts a + * fresh outermost evaluation of its receiver chain) -- ~1.5k in total. + * Unmemoized: ~2^LINKS per call node, ~100k in total. */ + ASSERT(work < 10000); + cbm_free_result(r); + PASS(); +} + +/* Deterministic complexity gate for the C/C++ resolver (O9: no clock): 4x the + * definitions and calls must cost ~4x the counted work (scope-frame probes, + * registry lookup probes, enclosing-definition candidates, real + * evaluations). Methods are DEFINED (out of class), so they are registered + * before the registry is indexed. Declaration-only methods are registered by + * the walk itself, after indexing, and are still found by a linear tail scan + * -- a separate, known quadratic (see the #1527 follow-up), not measured here. */ +static uint64_t clsp_member_calls_work(int n, int *out_real) { + size_t sz = (size_t)n * 128 + 512; + char *src = malloc(sz); + if (!src) + return 0; + size_t pos = (size_t)snprintf(src, sz, "struct S {\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " S& m%d();\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "};\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, "S& S::m%d() { return *this; }\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "void use() {\n S s;\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " s.m%d().m%d();\n", i, (i + 1) % n); + snprintf(src + pos, sz - pos, "}\n"); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + *out_real = r ? count_real_resolved(r, ".S.m") : -1; + if (r) + cbm_free_result(r); + return work; +} + +TEST(clsp_issue1527_scale_work_is_linear) { + int real1 = 0, real4 = 0; + uint64_t w1 = clsp_member_calls_work(1000, &real1); + uint64_t w4 = clsp_member_calls_work(4000, &real4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (real=%d) 4000=%llu (real=%d) ratio=%.2fx\n", + (unsigned long long)w1, real1, (unsigned long long)w4, real4, ratio); + ASSERT_EQ(real1, 2 * 1000); + ASSERT_EQ(real4, 2 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +/* A reachable post-allocation-failure state, pinned without memory pressure. + * The old recovery path returned a full table here. Check capacity BEFORE + * any missing-name lookup, and free the arena before the predicted RED. */ +TEST(scope_issue1527_recovery_sizes_from_all_bindings) { + CBMArena a; + cbm_arena_init(&a); + CBMScope *scope = cbm_scope_push(&a, NULL); + char names[64][24]; + bool setup_ok = scope != NULL; + for (int i = 0; i < 63 && setup_ok; i++) { + snprintf(names[i], sizeof(names[i]), "binding_%d", i); + setup_ok = cbm_scope_bind_callable_checked(scope, names[i], cbm_type_unknown(), "old"); + } + const CBMVarBinding *stable = setup_ok ? cbm_scope_lookup_local(scope, names[7]) : NULL; + if (setup_ok) + memset(&scope->index, 0, sizeof(scope->index)); + snprintf(names[63], sizeof(names[63]), "binding_63"); + bool bound = + setup_ok && cbm_scope_bind_callable_checked(scope, names[63], cbm_type_unknown(), "old"); + bool spare = bound && scope->binding_count == 64 && scope->index.cap >= 128; + bool preserved = false; + if (spare) { + preserved = cbm_scope_lookup_local(scope, "absent") == NULL; + for (int i = 0; i < 64; i++) + preserved = preserved && cbm_scope_lookup_local(scope, names[i]) != NULL; + preserved = preserved && + cbm_scope_bind_callable_checked(scope, names[7], cbm_type_unknown(), "new") && + cbm_scope_lookup_local(scope, names[7]) == stable && stable->callable_qn && + strcmp(stable->callable_qn, "new") == 0 && scope->binding_count == 64; + CBMScope *child = cbm_scope_push(&a, scope); + preserved = preserved && child && + cbm_scope_bind_callable_checked(child, names[7], cbm_type_unknown(), "child"); + const CBMVarBinding *shadow = child ? cbm_scope_lookup_binding(child, names[7]) : NULL; + preserved = preserved && shadow && strcmp(shadow->callable_qn, "child") == 0 && + strcmp(stable->callable_qn, "new") == 0; + } + cbm_arena_destroy(&a); + ASSERT_TRUE(setup_ok); + ASSERT_TRUE(bound); + ASSERT_TRUE(spare); + ASSERT_TRUE(preserved); + PASS(); +} + +/* Initial allocation, second allocation, and later growth fail independently + * of the arena. The failure must be explicit, not an uncached exponential walk. */ +TEST(clsp_issue1527_memo_allocation_failure_is_reported) { + char *src = clsp_chain_source(40, 1); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int successful_allocations = 0; successful_allocations <= 2; successful_allocations++) { + (void)cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(successful_allocations); + CBMFileResult *r = extract_cpp(src); + uint32_t walks = cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0 && walks == 1; + if (r) { + /* The allocation-free diagnostic also survives result compaction. */ + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && !fresh->lsp_skipped && + count_real_resolved(fresh, ".Tester.") == 41; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +extern const TSLanguage *tree_sitter_cpp(void); + +TEST(clsp_issue1527_memo_and_depth_failures_are_sticky) { + const char *src = "void f() { x; }"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_cpp()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode fn = ts_node_named_child(ts_tree_root_node(tree), 0); + TSNode body = ts_node_child_by_field_name(fn, "body", 4); + TSNode expr = ts_node_named_child(ts_node_named_child(body, 0), 0); + bool identifier = !ts_node_is_null(expr) && strcmp(ts_node_type(expr), "identifier") == 0; + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMResolvedCallArray out = {0}; + CLSPContext ctx; + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + const CBMType *first = cbm_type_named(&a, "test.First"); + const CBMType *second = cbm_type_named(&a, "test.Second"); + cbm_scope_bind(ctx.current_scope, "x", first); + ctx.eval_depth = 257; + bool depth_unknown = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A depth-truncated walk is incomplete; only a fresh walk may resume. */ + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + cbm_scope_bind(ctx.current_scope, "x", first); + bool shallow_ok = c_eval_expr_type(&ctx, expr) == first && !ctx.eval_failure; + cbm_scope_bind(ctx.current_scope, "x", second); + bool rebound = c_eval_expr_type(&ctx, expr) == second; + /* Force a new memo's allocation to fail; old slots remain arena-owned. */ + ctx.eval_memo_cap = ctx.eval_memo_count = 0; + ctx.eval_memo = NULL; + ctx.eval_memo_used = NULL; + ctx.test_memo_allocations_left = 0; + bool failed = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + bool stopped = true; + for (int i = 0; i < 20; i++) + stopped = stopped && cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + uint64_t later_work = cbm_lsp_work_take(); + cbm_arena_destroy(&a); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(identifier); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(shallow_ok); + ASSERT_TRUE(rebound); + ASSERT_TRUE(failed); + ASSERT_TRUE(stopped); + ASSERT_EQ(later_work, 0); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_memo_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_MEMO_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_depth_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_cross_file_memo_failures_propagate) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + bool failed[3] = {false}, recovered[3] = {false}; + bool shared_ready = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMTypeRegistry *shared = cbm_c_build_cross_registry(&a, defs, 2); + shared_ready = shared_ready && shared && shared->read_only; + reg.fallback = shared; /* the production prebuilt path uses a writable overlay */ + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .cpp_mode = true, + .defs = defs, + .def_count = 2}; + cbm_c_lsp_test_memo_fail_after(0); + bool ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, + NULL, NULL, 0, NULL, &out) + : mode == 1 + ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", true, + ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + cbm_c_lsp_test_memo_fail_after(-1); + failed[mode] = !ok && out.count == 0; + ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, NULL, + NULL, 0, NULL, &out) + : mode == 1 ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", + true, ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + recovered[mode] = ok && out.count > 0; + cbm_arena_destroy(&a); + } + ASSERT_TRUE(shared_ready); + for (int i = 0; i < 3; i++) { + ASSERT_TRUE(failed[i]); + ASSERT_TRUE(recovered[i]); + } + PASS(); +} + +TEST(registry_issue1527_index_preserves_registration_order) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMRegisteredType t = {.qualified_name = "test.S", .short_name = "S"}; + cbm_registry_add_type(®, t); + cbm_registry_add_type(®, t); + CBMRegisteredFunc f = { + .qualified_name = "test.S.m", .short_name = "m", .receiver_type = "test.S"}; + cbm_registry_add_func(®, f); + cbm_registry_add_func(®, f); + bool linear_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = true; + cbm_registry_finalize(®); + bool indexed_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = false; + cbm_registry_finalize(®); + bool default_last = cbm_registry_lookup_type(®, "test.S") == ®.types[1] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[1] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[1]; + f.qualified_name = "test.S.tail"; + f.short_name = "tail"; + cbm_registry_add_func(®, f); + bool tail = cbm_registry_lookup_func(®, "test.S.tail") == ®.funcs[2] && + cbm_registry_lookup_method(®, "test.S", "tail") == ®.funcs[2]; + cbm_arena_destroy(&a); + ASSERT_TRUE(linear_first); + ASSERT_TRUE(indexed_first); + ASSERT_TRUE(default_last); + ASSERT_TRUE(tail); + PASS(); +} + +TEST(clsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + cbm_c_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_cpp(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_c_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2, + .cpp_mode = true}; + if (mode == 2) { + correct = !cbm_batch_c_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_c_lsp_cross_status(&arena, src, (int)strlen(src), "test", true, + defs, 2, NULL, NULL, 0, NULL, &out) + : cbm_run_c_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", true, &overlay, NULL, + NULL, 0, NULL, &out); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_CPP, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_c_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(c_lsp) { + RUN_TEST(clsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(scope_issue1527_recovery_sizes_from_all_bindings); + RUN_TEST(clsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(clsp_issue1527_memo_and_depth_failures_are_sticky); + RUN_TEST(clsp_issue1527_preprocessed_memo_failure_is_reported); + RUN_TEST(clsp_issue1527_preprocessed_depth_failure_is_reported); + RUN_TEST(registry_issue1527_index_preserves_registration_order); + RUN_TEST(clsp_issue1527_cross_file_memo_failures_propagate); RUN_TEST(clsp_c_reassigned_function_pointer_calls_join_exact_occurrences); RUN_TEST(clsp_c_nested_function_pointer_shadow_restores_outer_target); RUN_TEST(clsp_cpp_repeated_same_leaf_calls_join_exact_occurrences); @@ -17264,4 +17739,6 @@ SUITE(c_lsp) { RUN_TEST(clsp_dll_multiple_functions); RUN_TEST(clsp_dll_func_ptr_typedef); RUN_TEST(clsp_easy_win_sfinaeconditional_return); + RUN_TEST(clsp_issue1527_fluent_chain_work_is_polynomial); + RUN_TEST(clsp_issue1527_scale_work_is_linear); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 4980426a92..cf35289210 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8919,6 +8919,51 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) { PASS(); } +/* #1527: the call-context metrics find each call's innermost enclosing + * Function/Method with a line sweep instead of scanning every def per call. + * The sweep must give the scan's exact answer -- smallest span, ties to the + * lowest def index -- on nested, overlapping, same-line, malformed + * (end < start) and unnamed/non-callable ranges. Fixed-seed generator: + * deterministic, many shapes. */ +static uint32_t enc_rand(uint32_t *state) { + *state = (*state * 1103515245u) + 12345u; + return (*state >> 16) & 0x7fffu; +} + +TEST(extract_enclosing_sweep_matches_scan_issue1527) { + static const char *const labels[] = {"Function", "Method", "Class", "Variable"}; + uint32_t seed = 1527; + for (int round = 0; round < 40; round++) { + CBMArena a; + cbm_arena_init(&a); + CBMFileResult r; + memset(&r, 0, sizeof(r)); + int ndefs = round == 0 ? 0 : (int)(enc_rand(&seed) % 300); + int ncalls = (int)(enc_rand(&seed) % 1500) + 1; + int max_line = (int)(enc_rand(&seed) % 400) + 5; + for (int i = 0; i < ndefs; i++) { + CBMDefinition d; + memset(&d, 0, sizeof(d)); + d.name = (enc_rand(&seed) % 10) == 0 ? NULL : "f"; + d.label = labels[enc_rand(&seed) % 4]; + d.start_line = 1 + (enc_rand(&seed) % (uint32_t)max_line); + int len = (int)(enc_rand(&seed) % 60) - 3; /* a few malformed end < start */ + d.end_line = (uint32_t)((int)d.start_line + len > 0 ? (int)d.start_line + len : 0); + cbm_defs_push(&r.defs, &a, d); + } + for (int i = 0; i < ncalls; i++) { + CBMCall c; + memset(&c, 0, sizeof(c)); + c.callee_name = "g"; + c.start_line = (int)(enc_rand(&seed) % (uint32_t)(max_line + 10)) - 2; + cbm_calls_push(&r.calls, &a, c); + } + ASSERT_EQ(cbm_test_enclosing_sweep_mismatches(&r), 0); + cbm_arena_destroy(&a); + } + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -9361,6 +9406,7 @@ SUITE(extraction) { RUN_TEST(json_toplevel_description_promoted_to_module_issue519); RUN_TEST(config_description_only_at_top_level_issue519); RUN_TEST(non_config_language_module_has_no_promoted_description_issue519); + RUN_TEST(extract_enclosing_sweep_matches_scan_issue1527); cbm_shutdown(); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9e26829e27..1fca364668 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10,7 +10,10 @@ #include "test_helpers.h" #include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test) #include "pipeline/pipeline.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "pipeline/lsp_surface.h" +#include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline_internal.h" #include "pipeline/artifact.h" #include "store/store.h" @@ -8166,6 +8169,168 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* Real cross-file memo failure, after successful extraction. The failure + * must reach the public per-file error list on both production drivers. */ +static int pipeline_lsp_failure_case(bool parallel, const char *stage, bool python, bool depth) { + const char *src = "struct S { int m() { return 1; } };\n" + "void run() { S s; s.m(); }\n"; + if (python) + src = "class S:\n def m(self):\n return 1\ndef run():\n s = S()\n s.m()\n"; + const char *filename = python ? "memo.py" : "memo.cpp"; + if (setup_usages_repo(filename, src, NULL, NULL) != 0) + FAIL("failed to create memo fixture"); + char path[512]; + if (parallel) { + for (int i = 0; i < 51; i++) { + snprintf(path, sizeof(path), "%s/memo_pad_%02d.%s", g_usages_tmpdir, i, + python ? "py" : "cpp"); + if (th_write_file(path, python ? "# inert parallel routing fixture\n" + : "// inert parallel routing fixture\n") != 0) { + teardown_usages_repo(); + FAIL("failed to create parallel routing fixture"); + } + } + } + const char *keys[] = { + "CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + depth ? (python ? "CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE" : "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE") + : (python ? "CBM_TEST_PY_LSP_MEMO_FAIL_STAGE" : "CBM_TEST_C_LSP_MEMO_FAIL_STAGE")}; + char *saved[3] = {0}; + bool env_saved = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + env_saved = env_saved && (!old || saved[i]); + } + if (!env_saved) { + for (int i = 0; i < 3; i++) + free(saved[i]); + teardown_usages_repo(); + FAIL("failed to save memo fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= cbm_setenv(keys[2], stage, 1); + snprintf(path, sizeof(path), "%s/memo.db", g_usages_tmpdir); + cbm_pipeline_t *p = set_rc == 0 ? cbm_pipeline_new(g_usages_tmpdir, path, CBM_MODE_FULL) : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(p, &errors, &error_count); + int memo_errors = 0; + const char *expected_phase = strcmp(stage, "cross") == 0 ? "lsp_skipped" : "extract"; + bool precise = true; + for (int i = 0; i < error_count; i++) { + if (errors[i].path && strcmp(errors[i].path, filename) == 0) { + memo_errors++; + precise = precise && errors[i].phase && strcmp(errors[i].phase, expected_phase) == 0 && + errors[i].reason && + strcmp(errors[i].reason, + depth ? (python ? CBM_PY_LSP_DEPTH_ERROR : CBM_C_LSP_DEPTH_ERROR) + : (python ? CBM_PY_LSP_MEMO_ERROR : CBM_C_LSP_MEMO_ERROR)) == 0; + } + } + cbm_pipeline_free(p); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + teardown_usages_repo(); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(memo_errors, 1); + ASSERT_TRUE(precise); + PASS(); +} + +TEST(pipeline_issue1527_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, false); +} +TEST(pipeline_issue1527_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, false); +} + +TEST(pipeline_issue1527_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, false); +} +TEST(pipeline_issue1527_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, false); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, true); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, true); +} + +TEST(pipeline_issue1527_cross_memo_failure_fallback) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_c_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_CPP, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + cbm_c_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16410,6 +16575,23 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { #endif SUITE(pipeline) { + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_fallback); RUN_TEST(pipeline_nested_fixture_files_are_written); RUN_TEST(pipeline_fixture_file_parent_is_preserved); RUN_TEST(pipeline_fixture_node_count_requires_exact_source_nodes); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index d65341900d..35dfe347b5 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -10,6 +10,7 @@ #include "test_framework.h" #include "cbm.h" #include "lsp/py_lsp.h" +#include "lsp/lsp_work.h" #include "pipeline/lsp_resolve.h" #include "pipeline/pass_lsp_cross.h" @@ -2223,16 +2224,30 @@ TEST(pylsp_issue710_heterogeneous_receiver_chain) { PASS(); } -TEST(pylsp_eval_steps_budget_degrades_gracefully) { - /* PY_EVAL_MAX_STEPS_PER_FILE guard: a file whose expressions demand - * more evaluator work than the per-file budget must still complete - * quickly and keep everything resolved BEFORE exhaustion — graceful - * partial degradation, not a hang, crash, or corrupted result. 90 - * statements x 30-link chains re-evaluated across the bind + emit - * phases (the assignment bind flushes the memo generation in between) - * comfortably exceed the 10000-step budget. */ - enum { BUDGET_STMTS = 90, BUDGET_LINKS = 30 }; - size_t sz = 4096 + (size_t)BUDGET_STMTS * (32 + (size_t)BUDGET_LINKS * 8); +/* Resolved INVOCATION rows whose callee QN ends with `suffix` (e.g. ".compile"). */ +static int count_resolved_suffix(const CBMFileResult *r, const char *suffix) { + size_t sl = strlen(suffix); + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + size_t ql = rc->callee_qn ? strlen(rc->callee_qn) : 0; + if (rc->kind == CBM_RESOLVED_INVOCATION && ql >= sl && + strcmp(rc->callee_qn + ql - sl, suffix) == 0) { + n++; + } + } + return n; +} + +TEST(pylsp_issue1527_chain_statements_resolve_completely) { + /* #1527 replaces the old budget-degradation guard: with no per-file + * evaluation budget, the #710 chain shape -- 90 statements x 30-link + * chains, re-evaluated across the bind + emit phases (the assignment bind + * flushes the memo generation in between) -- must still finish quickly + * AND resolve EVERY link and EVERY final call (each is a distinct call + * site), not merely the first statements. */ + enum { CHAIN_STMTS = 90, CHAIN_LINKS = 30 }; + size_t sz = 4096 + (size_t)CHAIN_STMTS * (32 + (size_t)CHAIN_LINKS * 8); char *src = malloc(sz); ASSERT_NOT_NULL(src); char *p = src; @@ -2247,11 +2262,11 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { "def run():\n"); p += n; left -= (size_t)n; - for (int s = 0; s < BUDGET_STMTS; s++) { + for (int s = 0; s < CHAIN_STMTS; s++) { n = snprintf(p, left, " v%d = G()", s); p += n; left -= (size_t)n; - for (int l = 0; l < BUDGET_LINKS; l++) { + for (int l = 0; l < CHAIN_LINKS; l++) { n = snprintf(p, left, ".add(%d)", l); p += n; left -= (size_t)n; @@ -2264,15 +2279,432 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { CBMFileResult *r = extract_py(src); free(src); ASSERT_NOT_NULL(r); - /* The first statements run with budget to spare — they must resolve. */ - ASSERT_GTE(require_resolved(r, "run", "G.compile"), 0); + ASSERT_EQ(count_resolved_suffix(r, ".G.compile"), CHAIN_STMTS); + ASSERT_EQ(count_resolved_suffix(r, ".G.add"), CHAIN_STMTS * CHAIN_LINKS); + cbm_free_result(r); + PASS(); +} + +TEST(pylsp_issue1527_large_module_resolves_every_call) { + /* #1527 reproduction: a module of 8000 classes plus one function calling + * `ClsN().chain().chain().method()` for each. Before the fix the per-file + * step cap stopped resolution after ~3333 statements (CALLS 14,666 of + * 24,000 in the graph) while the file still reported status `indexed`. + * Every one of the 8000 x 3 method call sites must resolve. */ + enum { N_CLASSES = 8000 }; + size_t sz = (size_t)N_CLASSES * 192 + 1024; + char *src = malloc(sz); + ASSERT_NOT_NULL(src); + size_t pos = (size_t)snprintf(src, sz, "from typing import Self\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + } + pos += (size_t)snprintf(src + pos, sz - pos, "def use():\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, " Cls%d().chain().chain().method()\n", i); + } + ASSERT(pos < sz); + CBMFileResult *r = + cbm_extract_file(src, (int)pos, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + free(src); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_resolved_suffix(r, ".method"), N_CLASSES); + ASSERT_EQ(count_resolved_suffix(r, ".chain"), 2 * N_CLASSES); + /* The last class is resolved exactly, not just some class. */ + ASSERT_GTE(require_resolved(r, "use", "Cls7999.method"), 0); cbm_free_result(r); PASS(); } /* ── Suite ─────────────────────────────────────────────────────── */ +static char *pylsp_work_fixture(int n_classes, int *out_len) { + /* Per class: ~140 chars (5-line def). Per call: ~50 chars. Overhead + * for the class number digits scales with log10(n) but the constant + * 256 covers up to 9-digit indices comfortably. */ + int approx = n_classes * 256 + 1024; + char *buf = (char *)malloc((size_t)approx); + if (!buf) + return NULL; + int pos = 0; + pos += snprintf(buf + pos, (size_t)(approx - pos), "from typing import Self\n"); + for (int i = 0; i < n_classes; i++) { + int n = snprintf(buf + pos, (size_t)(approx - pos), + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + if (n < 0 || pos + n >= approx) + break; + pos += n; + } + int n = snprintf(buf + pos, (size_t)(approx - pos), "def use():\n"); + pos += n; + for (int i = 0; i < n_classes; i++) { + int m = snprintf(buf + pos, (size_t)(approx - pos), + " Cls%d().chain().chain().method()\n", i); + if (m < 0 || pos + m >= approx) + break; + pos += m; + } + *out_len = pos; + return buf; +} + +/* Deterministic complexity gate (#1527, O9: no clock). The resolver + * primitives count every candidate they examine -- scope-frame probes, + * registry lookup probes, enclosing-definition candidates -- plus every real + * (non-memoized) expression evaluation, into a seam-build work counter. 4x + * the input must cost about 4x the work: a linear-scan lookup or a lost memo + * lands at ~16x (quadratic) or worse. Also asserts every call site resolved, + * so the work is not "linear" because resolution stopped early. */ +static uint64_t work_for(int n_classes, int *out_resolved) { + int slen = 0; + char *src = pylsp_work_fixture(n_classes, &slen); + if (!src) + return 0; + (void)cbm_lsp_work_take(); + CBMFileResult *r = + cbm_extract_file(src, slen, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + uint64_t work = cbm_lsp_work_take(); + *out_resolved = r ? r->resolved_calls.count : -1; + if (r) + cbm_free_result(r); + free(src); + return work; +} + +TEST(pylsp_scale_work_is_linear) { + int r1 = 0, r4 = 0; + uint64_t w1 = work_for(1000, &r1); + uint64_t w4 = work_for(4000, &r4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (resolved=%d) 4000=%llu (resolved=%d) ratio=%.2fx " + "(linear ~4x, quadratic ~16x)\n", + (unsigned long long)w1, r1, (unsigned long long)w4, r4, ratio); + /* Each class line: ClsN() + chain() + chain() + method() = 4 resolved sites. */ + ASSERT_EQ(r1, 4 * 1000); + ASSERT_EQ(r4, 4 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +extern const TSLanguage *tree_sitter_python(void); + +TEST(pylsp_issue1527_memo_allocation_failure_is_reported) { + int len = 0; + char *src = pylsp_work_fixture(200, &len); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int fail_after = 0; fail_after <= 1; fail_after++) { + cbm_py_lsp_test_memo_fail_after(fail_after); + CBMFileResult *r = extract_py(src); + cbm_py_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0; + if (r) { + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +TEST(pylsp_issue1527_memo_and_depth_failures_stop_work) { + const char *src = "x\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode node = ts_node_named_child(ts_node_named_child(ts_tree_root_node(tree), 0), 0); + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + const CBMType *known = cbm_type_named(&arena, "test.Known"); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.eval_depth = 256; + bool depth_unknown = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A fresh walk can resolve this shallow expression; the failed one cannot. */ + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + bool depth_recovered = cbm_py_lsp_test_eval(&ctx, node) == known && !ctx.eval_failure; + bool stopped = true; + for (int capacity_failure = 0; capacity_failure < 2; capacity_failure++) { + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.test_memo_allocations_left = 0; + /* Pin the maximum-capacity refusal without allocating a huge table. + * A NULL table bypasses lookup; insertion must report failed growth. */ + if (capacity_failure) + ctx.type_cache_cap = 1 << 26; + stopped = + stopped && cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + for (int i = 0; i < 20; i++) + stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && stopped; + stopped = stopped && cbm_lsp_work_take() == 0; + } + cbm_arena_destroy(&arena); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(depth_recovered); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class) { + const char *src = "class Local:\n pass\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse root-name fixture"); + } + bool closed = true, control = false; + for (int failure = 0; failure <= 2; failure++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.Local", .short_name = "Local"}); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.External", .short_name = "External"}); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + ctx.test_root_name_failure = failure; + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool local_absent = !cbm_scope_lookup_local(ctx.current_scope, "Local"); + bool external_present = cbm_scope_lookup_local(ctx.current_scope, "External") != NULL; + if (failure) + closed = + closed && ctx.callable_value_proof_disabled && local_absent && !external_present; + else + control = !ctx.callable_value_proof_disabled && local_absent && external_present; + cbm_arena_destroy(&arena); + } + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(control); + ASSERT_TRUE(closed); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows) { + char src[4096]; + size_t pos = (size_t)snprintf(src, sizeof(src), "def run(s: S):\n s + s\n"); + for (int i = 0; i < 200; i++) + pos += (size_t)snprintf(src + pos, sizeof(src) - pos, " s.m()\n"); + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.__add__", + .short_name = "__add__", + .label = "Method", + .receiver_type = "test.S", + .return_types = "S", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMResolvedCall prior = {.caller_qn = "test.run", + .callee_qn = "test.S.__add__", + .kind = CBM_RESOLVED_INVOCATION, + .confidence = 0.1f, + .strategy = "prior.strategy", + .reason = "prior.reason", + .site_start_byte = (uint32_t)(strstr(src, "s + s") - src), + .site_end_byte = (uint32_t)(strstr(src, "s + s") - src) + 5}; + CBMCall prior_synthetic = {.callee_name = "completed.synthetic"}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + CBMCallArray synthetic = {.items = &prior_synthetic, .count = 1, .cap = 1}; + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 3) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 3}; + cbm_py_lsp_test_memo_fail_after(1); + bool complete = + mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, NULL, + NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test", + &overlay, NULL, NULL, 0, NULL, &out, + &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + cbm_py_lsp_test_memo_fail_after(-1); + correct = correct && !complete && out.count == (mode == 2 ? 0 : 1) && + synthetic.count == 1 && prior.confidence == 0.1f && + strcmp(prior.strategy, "prior.strategy") == 0 && + strcmp(prior.reason, "prior.reason") == 0 && + strcmp(prior_synthetic.callee_name, "completed.synthetic") == 0; + if (mode != 2) + correct = correct && out.items[0].confidence == 0.1f && + strcmp(out.items[0].strategy, "prior.strategy") == 0 && + strcmp(out.items[0].reason, "prior.reason") == 0; + if (mode == 2) + out = (CBMResolvedCallArray){0}; + complete = mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, + NULL, NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + correct = correct && complete && out.count > 200 && + (mode == 2 || (synthetic.count > 1 && out.items[0].confidence > 0.1f)); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error) { + const char *src = "def run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_py_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_PYTHON, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + cbm_py_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(pylsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "class S:\n def m(self):\n return 1\ndef run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + cbm_py_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_py(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2}; + if (mode == 2) { + correct = !cbm_batch_py_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_py_lsp_cross_status(&arena, src, (int)strlen(src), "test", defs, + 2, NULL, NULL, 0, NULL, &out, NULL) + : cbm_run_py_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, + 0, NULL, &out, NULL); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_PYTHON, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_py_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error); + RUN_TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows); + RUN_TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class); + RUN_TEST(pylsp_issue1527_memo_and_depth_failures_stop_work); + RUN_TEST(pylsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(pylsp_scale_work_is_linear); /* Phase 2 — smoke */ RUN_TEST(pylsp_smoke_empty); RUN_TEST(pylsp_smoke_one_function); @@ -2366,5 +2798,6 @@ SUITE(py_lsp) { /* Evaluator guards — #710/#720 (distilled PRs #732 + #758) */ RUN_TEST(pylsp_issue710_deep_call_chain_resolves); RUN_TEST(pylsp_issue710_heterogeneous_receiver_chain); - RUN_TEST(pylsp_eval_steps_budget_degrades_gracefully); + RUN_TEST(pylsp_issue1527_chain_statements_resolve_completely); + RUN_TEST(pylsp_issue1527_large_module_resolves_every_call); } diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.c index f20e762753..1982e0623a 100644 --- a/tests/test_py_lsp_scale.c +++ b/tests/test_py_lsp_scale.c @@ -1,7 +1,7 @@ /* * test_py_lsp_scale.c — measure scaling behavior at 100 / 500 / 2000 - * classes-and-calls. Asserts that doubling the input doesn't more than - * 4x the runtime (catches accidental O(n^2) in the resolver). + * classes-and-calls. Timing is diagnostic; deterministic work is gated in + * the normal py_lsp suite. */ #include "test_framework.h" #include "cbm.h" @@ -88,26 +88,10 @@ TEST(pylsp_scale_linear_growth) { ASSERT(r_pct_100 > 0.5); ASSERT(r_pct_2000 > 0.5); - /* Quadratic-growth detector. 20x input: linear ~20-30x time, clear - * quadratic ~400x. The bound sits at 200x — mid-way in log space — so a - * real quadratic regression still fails by 2x while the CURRENT, KNOWN - * superlinear resolve curve does not produce false release blocks. - * - * The old bound of 100x was calibrated as "linear plus generous overhead", - * but the resolve path has never been linear here: measured 2026-08-11 - * (sanitized builds, deterministic across runs) — quiet arm64 host - * 57-68x, macos-15-intel CI runner 101.1x, per-function cost growing - * 0.5ms -> 1.7ms from 100 to 2000 functions. That ~O(n^1.4-1.5) curve is - * the audited short-name-lookup/negative-memo gap, tracked as #1527; this - * detector was flagging host CONSTANTS, not a complexity change. When - * #1527 lands, tighten this back down (~40x holds linear honestly). */ - if (t100 > 0.5) { // skip when t100 too small to compare reliably - double ratio = t2000 / t100; - printf(" scale ratio 2000/100: %.1fx (linear ~20x, known-superlinear ~60-100x, " - "quadratic ~400x)\n", - ratio); - ASSERT(ratio < 200.0); // flags clear quadratic; #1527 tracks the curve itself - } + /* Timing is diagnostic only. The normal py_lsp suite gates deterministic + * resolver work and complete resolution independently of host scheduling. */ + if (t100 > 0.0) + printf(" diagnostic scale ratio 2000/100: %.1fx\n", t2000 / t100); PASS(); } From 90132b5fe401d50bc5d2f3aceaabca5a3267bbae Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 20:38:20 +0200 Subject: [PATCH 4/6] fix(python): join module-level calls to exact LSP resolutions Preserve the arena-owned raw Module caller name so top-level resolutions join the extractor in both pipeline drivers. Add occurrence and ownership controls, including package modules and named lambdas. Signed-off-by: Martin Vogel --- internal/cbm/lsp/py_lsp.c | 4 +- tests/test_pipeline.c | 201 ++++++++++++++++++++++++++++++++++ tests/test_py_lsp.c | 219 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+), 1 deletion(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index e78bba7dcb..48cadcd45f 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -4817,7 +4817,9 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { uint32_t nc = rk.n; const char *prev_func = ctx->enclosing_func_qn; - ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.__module__", ctx->module_qn); + /* Extraction uses the file's Module QN at top level. Keep the caller + * arena-owned for cross-file APIs whose module_qn input may be temporary. */ + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 1fca364668..ba220bc84b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8331,6 +8331,205 @@ TEST(pipeline_issue1527_cross_memo_failure_fallback) { PASS(); } +static bool python_module_write_fixture_file(const char *root, const char *name, const char *text) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/%s", root, name); + return n > 0 && (size_t)n < sizeof(path) && th_write_file(path, text) == 0; +} + +/* An absent node, failed edge query, or heuristic edge must not satisfy this + * integration check. Exact source/target identities precede strategy checks. */ +static bool python_module_exact_lsp_edge(cbm_store_t *store, const char *project, + const char *caller, const char *caller_label, + const char *target, const char *target_label) { + char source_qn[768], target_qn[768]; + int sn = snprintf(source_qn, sizeof(source_qn), "%s.%s", project, caller); + int tn = snprintf(target_qn, sizeof(target_qn), "%s.%s", project, target); + if (sn <= 0 || (size_t)sn >= sizeof(source_qn) || tn <= 0 || (size_t)tn >= sizeof(target_qn)) + return false; + cbm_node_t source = {0}, destination = {0}; + bool correct = + cbm_store_find_node_by_qn(store, project, source_qn, &source) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, target_qn, &destination) == CBM_STORE_OK && + source.id > 0 && destination.id > 0 && source.label && destination.label && + strcmp(source.label, caller_label) == 0 && strcmp(destination.label, target_label) == 0; + cbm_edge_t *edges = NULL; + int edge_count = 0, hits = 0; + if (correct) { + correct = cbm_store_find_edges_by_source_type(store, source.id, "CALLS", &edges, + &edge_count) == CBM_STORE_OK; + for (int i = 0; correct && i < edge_count; i++) { + if (edges[i].source_id != source.id || edges[i].target_id != destination.id) + continue; + hits++; + const char *properties = edges[i].properties_json; + yyjson_doc *doc = properties ? yyjson_read(properties, strlen(properties), 0) : NULL; + const char *strategy = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "strategy")) : NULL; + correct = strategy && strncmp(strategy, "lsp_", 4) == 0; + if (doc) + yyjson_doc_free(doc); + } + } + cbm_store_free_edges(edges, edge_count); + cbm_node_free_fields(&source); + cbm_node_free_fields(&destination); + return correct && hits == 1; +} + +static int python_module_callers_pipeline_case(bool parallel) { + static const char helpers[] = "def helper():\n" + " return 1\n" + "class Service:\n" + " def run(self):\n" + " return 2\n"; + static const char app[] = "from helpers import helper, Service\n" + "def local_helper():\n" + " return 3\n" + "class Local:\n" + " def run(self):\n" + " return 4\n" + "class Controller:\n" + " def run(self):\n" + " helper()\n" + " method_service = Service()\n" + " method_service.run()\n" + "def control():\n" + " helper()\n" + " function_service = Service()\n" + " function_service.run()\n" + " local_helper()\n" + " function_local = Local()\n" + " function_local.run()\n" + "helper()\n" + "module_service = Service()\n" + "module_service.run()\n" + "local_helper()\n" + "module_local = Local()\n" + "module_local.run()\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } expected[] = { + {"app", "Module", "helpers.helper", "Function"}, + {"app", "Module", "helpers.Service.run", "Method"}, + {"app", "Module", "app.local_helper", "Function"}, + {"app", "Module", "app.Local.run", "Method"}, + {"app.control", "Function", "helpers.helper", "Function"}, + {"app.control", "Function", "helpers.Service.run", "Method"}, + {"app.control", "Function", "app.local_helper", "Function"}, + {"app.control", "Function", "app.Local.run", "Method"}, + {"app.Controller.run", "Method", "helpers.helper", "Function"}, + {"app.Controller.run", "Method", "helpers.Service.run", "Method"}, + {"pkg.__init__", "Module", "helpers.helper", "Function"}, + }; + char root[512]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-module-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python module fixture"); + char path[768]; + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file(root, "helpers.py", helpers) && + python_module_write_fixture_file(root, "app.py", app) && + python_module_write_fixture_file(root, "pkg/__init__.py", + "from helpers import helper\n" + "PACKAGE_VALUE = helper()\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "module_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "padding = 0\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python module fixture"); + } + + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {0}; + bool saved_ok = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + free(saved[0]); + free(saved[1]); + th_rmtree(root); + FAIL("failed to save Python module fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + n = snprintf(path, sizeof(path), "%s/module.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + int file_count = 0; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (parallel ? 55 : 3); + cbm_store_free_nodes(files, file_count); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + bool edge = project && python_module_exact_lsp_edge( + store, project, expected[i].caller, expected[i].caller_label, + expected[i].target, expected[i].target_label); + correct = correct && edge; + } + /* Neither the ordinary nor package Module should acquire a fake + * __module__ owner node. A failed lookup query is not absence proof. */ + const char *phantoms[] = {"app.__module__", "pkg.__init__.__module__"}; + for (int i = 0; project && i < 2; i++) { + char qn[768]; + n = snprintf(qn, sizeof(qn), "%s.%s", project, phantoms[i]); + cbm_node_t node = {0}; + bool absent = + n > 0 && (size_t)n < sizeof(qn) && + cbm_store_find_node_by_qn(store, project, qn, &node) == CBM_STORE_NOT_FOUND; + correct = correct && absent; + cbm_node_free_fields(&node); + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, 0); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pipeline_python_module_callers_sequential) { + return python_module_callers_pipeline_case(false); +} + +TEST(pipeline_python_module_callers_parallel) { + return python_module_callers_pipeline_case(true); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16575,6 +16774,8 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { #endif SUITE(pipeline) { + RUN_TEST(pipeline_python_module_callers_sequential); + RUN_TEST(pipeline_python_module_callers_parallel); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 35dfe347b5..adc84913b3 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -2697,7 +2697,226 @@ TEST(pylsp_issue1527_depth_failure_reports_all_entry_points) { PASS(); } +/* Join one known source occurrence, not merely two equal aggregate counts. + * Changing the owner or range must not borrow a neighbouring resolution. */ +static bool pylsp_module_site_joins(const CBMCallArray *calls, const CBMResolvedCallArray *resolved, + const char *source, const char *site_text, const char *caller, + const char *target) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source); + uint32_t end = start + (uint32_t)strlen(site_text); + const CBMCall *call = NULL; + int call_count = 0; + for (int i = 0; i < calls->count; i++) { + const CBMCall *candidate = &calls->items[i]; + if (candidate->enclosing_func_qn && strcmp(candidate->enclosing_func_qn, caller) == 0 && + candidate->site_start_byte == start && candidate->site_end_byte == end) { + call = candidate; + call_count++; + } + } + if (call_count != 1) + return false; + const CBMResolvedCall *hit = cbm_pipeline_find_lsp_resolution(resolved, call, false); + if (!hit || !hit->caller_qn || strcmp(hit->caller_qn, caller) != 0 || !hit->callee_qn || + strcmp(hit->callee_qn, target) != 0 || hit->kind != CBM_RESOLVED_INVOCATION || + hit->site_start_byte != start || hit->site_end_byte != end || + hit->source_origin != call->source_origin) + return false; + int semantic_count = 0; + for (int i = 0; i < resolved->count; i++) { + const CBMResolvedCall *candidate = &resolved->items[i]; + if (candidate->kind == CBM_RESOLVED_INVOCATION && candidate->caller_qn && + strcmp(candidate->caller_qn, caller) == 0 && candidate->site_start_byte == start && + candidate->site_end_byte == end) + semantic_count++; + } + CBMCall wrong = *call; + wrong.enclosing_func_qn = "unrelated.module.caller"; + bool rejects_owner = cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; + wrong = *call; + wrong.site_start_byte++; + return semantic_count == 1 && rejects_owner && + cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; +} + +static bool pylsp_module_has_def(const CBMFileResult *r, const char *qn) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].qualified_name && strcmp(r->defs.items[i].qualified_name, qn) == 0) + return true; + } + return false; +} + +TEST(pylsp_module_callers_join_exact_raw_occurrences) { + static const char source[] = "class Alpha:\n" + " def render(self):\n" + " return 1\n" + "class Beta:\n" + " def render(self):\n" + " return 2\n" + "def helper():\n" + " return 3\n" + "def control():\n" + " function_a = Alpha()\n" + " function_a.render()\n" + " helper()\n" + "class Controller:\n" + " def control(self):\n" + " method_b = Beta()\n" + " method_b.render()\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.app") == 0; + if (correct) { + correct = + pylsp_module_has_def(r, "test.app.Alpha.render") && + pylsp_module_has_def(r, "test.app.Beta.render") && + pylsp_module_has_def(r, "test.app.control") && + pylsp_module_has_def(r, "test.app.Controller.control") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_a.render()", + "test.app", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_b.render()", + "test.app", "test.app.Beta.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "function_a.render()", + "test.app.control", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "method_b.render()", + "test.app.Controller.control", "test.app.Beta.render"); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + + /* Package symbol-scope normalization is separate. A builtin isolates + * this package Module's raw caller identity from that remaining issue. */ + static const char init_source[] = "PACKAGE_SIZE = len([1])\n"; + r = cbm_extract_file(init_source, (int)strlen(init_source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.pkg.__init__") == 0 && + pylsp_module_has_def(r, "builtins.len") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, init_source, "len([1])", + "test.pkg.__init__", "builtins.len"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair) { + static const char source[] = "class Number:\n" + " def __add__(self, other):\n" + " return self\n" + "module_number = Number()\n" + "add = lambda left, right: left + right\n" + "first = add(module_number, module_number)\n" + "second = add(module_number, module_number)\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + pylsp_module_has_def(r, "test.app.Number.__add__") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "left + right", + "test.app.", "test.app.Number.__add__"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites) { + static const char source[] = "from helpers import Alpha, Beta\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + static const char *const paths[] = {"app.py", "pkg/__init__.py"}; + static const char *const modules[] = {"test.app", "test.pkg.__init__"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.helpers.Alpha", + .short_name = "Alpha", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Alpha.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Alpha", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta", + .short_name = "Beta", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Beta", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *import_names[] = {"Alpha", "Beta"}; + const char *import_qns[] = {"test.helpers.Alpha", "test.helpers.Beta"}; + bool correct = true; + for (int path = 0; path < 2; path++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[path], 0, NULL, NULL); + if (!r || r->has_error || r->parse_incomplete || r->lsp_skipped || !r->module_qn || + strcmp(r->module_qn, modules[path]) != 0) { + correct = false; + if (r) + cbm_free_result(r); + continue; + } + for (int prebuilt = 0; prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = prebuilt ? cbm_py_build_cross_registry(&shared, defs, 4) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + char module_input[64]; + snprintf(module_input, sizeof(module_input), "%s", modules[path]); + CBMResolvedCallArray out = {0}; + CBMLSPStatus status = + prebuilt ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), module_input, &overlay, + import_names, import_qns, 2, NULL, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + module_input, defs, 4, import_names, + import_qns, 2, NULL, &out, NULL); + /* Returned caller strings must outlive the caller's input buffer. + * Overwrite live storage, avoiding a dangling-pointer test. */ + memset(module_input, 'x', strlen(module_input)); + correct = correct && (!prebuilt || (base && base->read_only)) && + status == CBM_LSP_COMPLETE && + pylsp_module_site_joins(&r->calls, &out, source, "module_a.render()", + modules[path], "test.helpers.Alpha.render") && + pylsp_module_site_joins(&r->calls, &out, source, "module_b.render()", + modules[path], "test.helpers.Beta.render"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_module_callers_join_exact_raw_occurrences); + RUN_TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair); + RUN_TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites); RUN_TEST(pylsp_issue1527_depth_failure_reports_all_entry_points); RUN_TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error); RUN_TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows); From af6af273475e046565d1c3e3d90a8f00bce199fb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 21:14:20 +0200 Subject: [PATCH 5/6] fix(python): separate package symbol scope from file identity Resolve symbols declared in __init__.py under their package while retaining the raw module identity for file and top-level caller nodes. Carry explicit language through registry callers, select Python package imports, preserve long exact names, and report scope allocation failures without partial output. Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 6 + internal/cbm/helpers.c | 11 + internal/cbm/lsp/py_lsp.c | 126 +++++++--- internal/cbm/lsp/py_lsp.h | 9 +- src/pipeline/pass_calls.c | 25 +- src/pipeline/pass_lsp_cross.c | 77 ++++-- src/pipeline/pass_parallel.c | 71 +++--- src/pipeline/pass_semantic.c | 26 +- src/pipeline/pass_usages.c | 18 +- src/pipeline/pipeline_internal.h | 9 + src/pipeline/registry.c | 105 +++++--- tests/test_pipeline.c | 356 +++++++++++++++++++++++++++ tests/test_py_lsp.c | 396 +++++++++++++++++++++++++++++++ tests/test_registry.c | 270 +++++++++++++++++++++ 14 files changed, 1367 insertions(+), 138 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6cf6d04fec..56f57dcd4a 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -456,6 +456,7 @@ typedef enum { CBM_LSP_COMPLETE = 0, CBM_LSP_MEMO_FAILED, CBM_LSP_DEPTH_EXCEEDED, + CBM_LSP_SCOPE_FAILED, } CBMLSPStatus; // Growable arrays used during extraction. @@ -982,4 +983,9 @@ bool cbm_label_is_relation(const char *label); // `label` may be NULL (returns false). Defined in helpers.c. bool cbm_label_is_registry_symbol(const char *label); +/* Python symbol scope only: callers must establish the language explicitly. + * A trailing .__init__ is omitted when a nonempty prefix precedes it; raw + * Module/file identity is unchanged. NULL returns zero. */ +size_t cbm_fqn_symbol_scope_len(const char *module_qn); + #endif // CBM_H diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 4c4d340f0b..4ab8ba48e7 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1611,6 +1611,17 @@ char *cbm_fqn_compute(CBMArena *a, const char *project, const char *rel_path, co return buf; } +size_t cbm_fqn_symbol_scope_len(const char *module_qn) { + if (!module_qn) + return 0; + size_t len = strlen(module_qn); + static const char suffix[] = ".__init__"; + size_t suffix_len = sizeof(suffix) - 1; + if (len > suffix_len && memcmp(module_qn + len - suffix_len, suffix, suffix_len) == 0) + return len - suffix_len; + return len; +} + char *cbm_fqn_module(CBMArena *a, const char *project, const char *rel_path) { return cbm_fqn_compute(a, project, rel_path, NULL); } diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 48cadcd45f..29cec01e73 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -39,6 +39,10 @@ #ifdef CBM_ENABLE_TEST_SEAMS static _Thread_local int py_test_memo_allocations_left = -1; static _Thread_local bool py_test_depth_failure; +static _Thread_local int py_test_scope_allocations_left = -1; +void cbm_py_lsp_test_scope_fail_after(int successful_allocations) { + py_test_scope_allocations_left = successful_allocations; +} void cbm_py_lsp_test_depth_fail(bool enabled) { py_test_depth_failure = enabled; } @@ -65,8 +69,9 @@ void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { result->has_error = true; result->lsp_skipped = true; /* Static lifetime also works when the result arena cannot allocate. */ - result->error_msg = - status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR : CBM_PY_LSP_MEMO_ERROR; + result->error_msg = status == CBM_LSP_SCOPE_FAILED ? CBM_PY_LSP_SCOPE_ERROR + : status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR + : CBM_PY_LSP_MEMO_ERROR; } // Forward decls @@ -290,9 +295,34 @@ static void py_scope_restore(PyLSPContext *ctx, CBMScope *saved) { ctx->current_scope = saved; } -void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, - const CBMTypeRegistry *registry, const char *module_qn, - CBMResolvedCallArray *out) { +/* Only a changed Python scope needs storage. Failure is explicit: never + * register or walk a package using the raw file scope as an OOM fallback. */ +static bool py_symbol_scope(CBMArena *arena, const char *module_qn, const char *stage, + const char **out) { + *out = module_qn; + size_t len = cbm_fqn_symbol_scope_len(module_qn); + if (!module_qn || module_qn[len] == '\0') + return true; +#ifdef CBM_ENABLE_TEST_SEAMS + const char *fail_stage = getenv("CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"); + if ((fail_stage && strcmp(fail_stage, stage) == 0) || py_test_scope_allocations_left == 0) + return false; + if (py_test_scope_allocations_left > 0) + py_test_scope_allocations_left--; +#else + (void)stage; +#endif + char *scope = cbm_arena_strndup(arena, module_qn, len); + if (!scope) + return false; + *out = scope; + return true; +} + +static void py_lsp_init_for_stage(PyLSPContext *ctx, CBMArena *arena, const char *source, + int source_len, const CBMTypeRegistry *registry, + const char *module_qn, CBMResolvedCallArray *out, + const char *stage) { if (!ctx) return; memset(ctx, 0, sizeof(PyLSPContext)); @@ -306,13 +336,23 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou ctx->source_len = source_len; ctx->registry = registry; ctx->registry_head = (CBMTypeRegistry *)registry; - ctx->module_qn = module_qn; + ctx->file_module_qn = module_qn; ctx->resolved_calls = out; + if (!py_symbol_scope(arena, module_qn, stage, &ctx->module_qn)) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return; + } ctx->current_scope = py_scope_push_checked(ctx); const char *dbg = getenv("CBM_LSP_DEBUG"); ctx->debug = dbg && dbg[0] && dbg[0] != '0'; } +void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, + const CBMTypeRegistry *registry, const char *module_qn, + CBMResolvedCallArray *out) { + py_lsp_init_for_stage(ctx, arena, source, source_len, registry, module_qn, out, "context"); +} + void py_lsp_add_import(PyLSPContext *ctx, const char *local_name, const char *module_qn) { if (!ctx || !local_name || !module_qn) return; @@ -2836,17 +2876,18 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // segment) and look up ".". A genuinely-external module // (requests, os) has no such project def, so it correctly stays // lsp_module_attr_unresolved. - if (mod && ctx->module_qn) { - const char *last_dot = strrchr(ctx->module_qn, '.'); - if (last_dot && last_dot > ctx->module_qn) { - size_t root_len = (size_t)(last_dot - ctx->module_qn); + if (mod && ctx->file_module_qn) { + const char *last_dot = strrchr(ctx->file_module_qn, '.'); + if (last_dot && last_dot > ctx->file_module_qn) { + size_t root_len = (size_t)(last_dot - ctx->file_module_qn); // Skip if mod is already rooted under the project to avoid // "..mod". - if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) { + if (!(strncmp(mod, ctx->file_module_qn, root_len) == 0 && + mod[root_len] == '.')) { char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); if (qual_mod) { - memcpy(qual_mod, ctx->module_qn, root_len); + memcpy(qual_mod, ctx->file_module_qn, root_len); qual_mod[root_len] = '.'; strcpy(qual_mod + root_len + 1, mod); const CBMRegisteredFunc *qf = @@ -4305,6 +4346,7 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) return; } size_t prefix_len = strlen(ctx->module_qn); + bool package_scope = ctx->file_module_qn && strcmp(ctx->file_module_qn, ctx->module_qn) != 0; CBMTypeShortIter all_types; cbm_registry_all_types_chain(ctx->registry, &all_types); for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { @@ -4315,6 +4357,8 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) cbm_scope_lookup_local(defined, name)) { continue; } + if (package_scope && strchr(qn + prefix_len + 1, '.')) + continue; /* submodule/nested classes are not package globals */ py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); } } @@ -4819,7 +4863,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { const char *prev_func = ctx->enclosing_func_qn; /* Extraction uses the file's Module QN at top level. Keep the caller * arena-owned for cross-file APIs whose module_qn input may be temporary. */ - ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->module_qn); + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->file_module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { @@ -5053,6 +5097,16 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, if (!arena || !result || result->lsp_skipped) return; + CBMTypeRegistry reg; + cbm_registry_init(®, arena); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, result->module_qn, + &result->resolved_calls, "raw"); + if (ctx.eval_failure) { + cbm_py_lsp_record_failure(result, ctx.eval_failure); + return; + } + /* Inject minimal builtin definitions as real graph nodes (builtins.len, * builtins.str, builtins.str.upper, ...). The typeshed registry already * RESOLVES builtin calls (emitting the strategy + a "builtins.*" callee_qn), @@ -5062,12 +5116,9 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, * builtin/constructor/method edges point at. Upsert dedups by QN. */ py_builtins_inject_defs(result, arena); - CBMTypeRegistry reg; - cbm_registry_init(®, arena); - cbm_python_stdlib_register(®, arena); - const char *module_qn = result->module_qn; + const char *module_qn = ctx.module_qn; // Register the file's own definitions so calls inside this file can // resolve via the registry. @@ -5088,8 +5139,6 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, reg.index_first_registered = true; cbm_registry_finalize_into(®, &idx_arena); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, &result->resolved_calls); /* Let the resolver inject synthetic syntactic calls for operator/subscript * dunder desugaring so those recovered calls reach the CALLS-edge pipeline. */ ctx.syn_calls = &result->calls; @@ -5187,7 +5236,7 @@ static void py_split_field_defs(CBMArena *arena, const char *field_defs, const c /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ -static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, +static bool py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, CBMLSPDef *defs, int def_count) { /* Pass 1: types only — the method pass probes the registry per Method def * (receiver auto-registration), which is a LINEAR scan pre-finalize: @@ -5251,7 +5300,13 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe ret_types[n] = NULL; } } - PySignatureParamParserContext parser_ctx = {.module_qn = d->def_module_qn}; + const char *symbol_scope = d->def_module_qn; + /* Fallback definitions can be mixed-language. Only authoritative + * Python metadata opts into package scope normalization. */ + if (d->lang == CBM_LANG_PYTHON && + !py_symbol_scope(arena, d->def_module_qn, "register", &symbol_scope)) + return false; + PySignatureParamParserContext parser_ctx = {.module_qn = symbol_scope}; const CBMType **param_types = cbm_type_materialize_signature_params( arena, d->signature_param_types, d->signature_param_count, py_signature_param_type_adapter, &parser_ctx); @@ -5271,8 +5326,16 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe cbm_registry_add_func(reg, rf); } } + return true; } +#ifdef CBM_ENABLE_TEST_SEAMS +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count) { + return py_register_lsp_defs(arena, NULL, reg, defs, def_count); +} +#endif + bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, @@ -5315,15 +5378,20 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in /* Index allocations go to a per-call scratch arena (see php_lsp_cross). */ CBMArena idx_arena; cbm_arena_init(&idx_arena); - py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, module_qn, out, "cross"); + if (ctx.eval_failure) + goto cross_cleanup; + if (!py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count)) { + ctx.eval_failure = CBM_LSP_SCOPE_FAILED; + goto cross_cleanup; + } py_mark_ambiguous_callable_bindings(®); /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" * comment for the rationale. */ cbm_registry_finalize_into(®, &idx_arena); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, out); ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { @@ -5332,6 +5400,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in } py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); +cross_cleanup: cbm_arena_destroy(&idx_arena); if (owns_tree && tree) @@ -5357,7 +5426,8 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i if (d->lang != CBM_LANG_PYTHON) continue; /* Reuse the existing register fn on a single-def slice (n=1 inline). */ - py_register_lsp_defs(arena, NULL, reg, d, 1); + if (!py_register_lsp_defs(arena, NULL, reg, d, 1)) + return NULL; /* caller owns the shared arena and other registries */ } py_mark_ambiguous_callable_bindings(reg); @@ -5401,7 +5471,9 @@ CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( TSNode root = ts_tree_root_node(tree); PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, reg, module_qn, out); + py_lsp_init_for_stage(&ctx, arena, source, source_len, reg, module_qn, out, "cross"); + if (ctx.eval_failure) + goto prebuilt_cleanup; ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { @@ -5410,7 +5482,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( } py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); - +prebuilt_cleanup: if (owns_tree && tree) ts_tree_delete(tree); if (parser) diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index e7536b8f9e..d733b31538 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -61,7 +61,8 @@ typedef struct { // caller QNs. const char *enclosing_func_qn; const char *enclosing_class_qn; - const char *module_qn; + const char *module_qn; // Python symbol scope (package __init__ omitted) + const char *file_module_qn; // raw file identity; borrowed for this API call // Output: resolved calls accumulate here. CBMResolvedCallArray *resolved_calls; @@ -102,7 +103,7 @@ typedef struct { uint32_t type_cache_gen; // bumped on every scope mutation (O(1) flush) int eval_depth; // evaluator recursion depth (PY_LSP_MAX_EVAL_DEPTH) uint32_t eval_truncations; // depth-cap cutoff count — gates memo inserts - CBMLSPStatus eval_failure; // sticky: memo/depth failure makes this walk incomplete + CBMLSPStatus eval_failure; // sticky: explicit failure makes this walk incomplete #ifdef CBM_ENABLE_TEST_SEAMS int test_memo_allocations_left; int test_root_name_failure; // 1 = NULL, 2 = empty; only the root-name collection @@ -167,10 +168,14 @@ void cbm_python_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); #define CBM_PY_LSP_DEPTH_ERROR "Python expression recursion depth limit exceeded" #define CBM_PY_LSP_MEMO_ERROR "Python expression memo allocation or capacity failure" +#define CBM_PY_LSP_SCOPE_ERROR "Python symbol scope allocation failure" void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); #ifdef CBM_ENABLE_TEST_SEAMS void cbm_py_lsp_test_memo_fail_after(int successful_allocations); void cbm_py_lsp_test_depth_fail(bool enabled); +void cbm_py_lsp_test_scope_fail_after(int successful_allocations); +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count); const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root); #endif diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f068ad4112..1c0dc1a62e 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -377,7 +377,8 @@ bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *proje /* Handle a route registration call: create Route node + HANDLES edge. */ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char route_qn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -397,8 +398,8 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca esc_fa); cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, - module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t hres = cbm_registry_resolve_lang( + ctx->registry, call->second_arg_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *handler = cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); if (handler != NULL) { @@ -588,10 +589,11 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count); + handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count, + lang); return; } if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { @@ -675,7 +677,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM res.strategy = lsp->strategy; res.candidate_count = 1; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, - imp_vals, imp_count, false); + imp_vals, imp_count, false, lang); return SKIP_ONE; } } @@ -741,8 +743,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM lang, lsp->strategy, lsp->callee_qn, ctx->project_name); cbm_resolution_t res = {0}; if (!rust_external) { - res = cbm_registry_resolve(ctx->registry, call->callee_name, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } if (!res.qualified_name || res.qualified_name[0] == '\0') { /* Resolution is empty when the callee belongs to an EXTERNAL client @@ -767,7 +769,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && call->first_string_arg[0] == '/') { handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count); + imp_count, lang); return SKIP_ONE; } cbm_svc_kind_t esvc = cbm_service_pattern_match(call->callee_name); @@ -888,7 +890,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, drop_plain_call); + imp_count, drop_plain_call, lang); return SKIP_ONE; } @@ -1125,7 +1127,8 @@ static int scan_depends_in_sig(cbm_pipeline_ctx_t *ctx, const cbm_regex_t *re, c } memcpy(func_ref, scan + match[SKIP_ONE].rm_so, (size_t)ref_len); func_ref[ref_len] = '\0'; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, func_ref, module_qn, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_ref, module_qn, ik, iv, + ic, CBM_LANG_PYTHON); if (res.qualified_name && res.qualified_name[0] != '\0') { const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_qn(ctx->gbuf, def->qualified_name); const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name); diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 6629956e7e..643d3ff191 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -227,11 +227,12 @@ static bool pxc_base_strategy_is_weak(const char *strategy) { * stdlib and third-party bases land here and keep their raw spelling. */ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { if (!reg || !raw || !raw[0]) { return NULL; } - cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = + cbm_registry_resolve_lang(reg, raw, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || !res.qualified_name[0]) { return NULL; } @@ -249,7 +250,8 @@ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *ra * registry does not know keeps working exactly as before. */ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, const cbm_registry_t *reg, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { if (!bases || !bases[0]) { return NULL; } @@ -264,7 +266,7 @@ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, } for (int i = 0; i < count; i++) { const char *qn = - pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count, lang); resolved[i] = qn ? qn : bases[i]; } resolved[count] = NULL; @@ -419,7 +421,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch * raw source spelling their own registrar already knows how to handle. */ dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, - imp_keys, imp_vals, imp_count) + imp_keys, imp_vals, imp_count, lang) : pxc_join_pipe(arena, src->base_classes); dst->signature_param_types = NULL; dst->signature_param_count = 0; @@ -622,8 +624,9 @@ static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result fields[f].type_qn = NULL; CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; - const char *qn = - name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + const char *qn = name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, + imp_count, CBM_LANG_PYTHON) + : NULL; if (!qn || !ft->field_name || !ft->field_name[0]) { continue; } @@ -2037,16 +2040,33 @@ static void pxc_mark_module_defs(const CBMModuleDefIndex *idx, bool *selected, * (`project.module`). Select the nearest materialized module prefix; the * language registry still has to prove the full target QN before an edge is * emitted, so this broadens the candidate set without weakening precision. */ -static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int pxc_import_probe_allocations_left = -1; +void cbm_pxc_test_import_probe_fail_after(int successful_allocations) { + pxc_import_probe_allocations_left = successful_allocations; +} +#endif + +static bool pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, const CBMLSPDef *all_defs, CBMLanguage caller_lang, const char *import_qn, int *total) { if (!idx || !idx->ht || !import_qn || !import_qn[0]) { - return; - } - char *candidate = strdup(import_qn); - if (!candidate) { - return; + return true; } + static const char init_seg[] = ".__init__"; + size_t import_len = strlen(import_qn); + if (import_len > SIZE_MAX - sizeof(init_seg)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (pxc_import_probe_allocations_left == 0) + return false; + if (pxc_import_probe_allocations_left > 0) + pxc_import_probe_allocations_left--; +#endif + char *candidate = (char *)malloc(import_len + sizeof(init_seg)); + if (!candidate) + return false; + memcpy(candidate, import_qn, import_len + 1); bool matched = false; for (;;) { pxc_module_entry_t *entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); @@ -2058,6 +2078,31 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, matched = true; break; } + if (caller_lang == CBM_LANG_PYTHON) { + size_t cand_len = strlen(candidate); + memcpy(candidate + cand_len, init_seg, sizeof(init_seg)); + entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); + candidate[cand_len] = '\0'; + /* A suffixed module may be a non-Python directory named __init__. + * Only this new probe filters strictly; normal-entry precedence + * above remains unchanged. Eligibility is independent of dedup. */ + bool eligible = false; + for (int j = 0; entry && j < entry->count; j++) { + int di = entry->indices[j]; + if (all_defs[di].lang != CBM_LANG_PYTHON) + continue; + eligible = true; + if (!selected[di]) { + selected[di] = true; + if (total) + (*total)++; + } + } + if (eligible) { + matched = true; + break; + } + } char *dot = strrchr(candidate, '.'); if (!dot) { break; @@ -2088,6 +2133,7 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, } } free(candidate); + return true; } CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) { @@ -2166,7 +2212,10 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef int total = 0; pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); for (int i = 0; i < imp_count; i++) { - pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total); + if (!pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total)) { + free(selected); + return NULL; /* out_success remains false: dispatcher uses all_defs */ + } } if (pxc_is_jvm_lang(caller_lang) && idx->namespace_ht) { const char *namespace_key = pxc_namespace_index_key(caller_namespace); diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 726199fde8..af415e13b3 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -573,9 +573,9 @@ static bool is_checked_exception(const char *name) { static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -2219,7 +2219,7 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou const CBMCall *call, const char *route_path, const char *handler_ref, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic) { + const char **ik, const char **iv, int ic, CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char rqn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -2238,7 +2238,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou esc_rp); cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); + cbm_resolution_t hres = + cbm_registry_resolve_lang(registry, handler_ref, module_qn, ik, iv, ic, lang); if (hres.qualified_name && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); if (h) { @@ -2531,7 +2532,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); const char *arg = call->first_string_arg; @@ -2558,7 +2559,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count); + registry, main_gbuf, imp_keys, imp_vals, imp_count, lang); return; } /* No path found — fall through to normal CALLS edge */ @@ -2996,8 +2997,8 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB lang, lsp->strategy, lsp->callee_qn, rc->project_name); if ((!res.qualified_name || !res.qualified_name[0]) && !call->requires_lsp_resolution && !rust_external) { - res = cbm_registry_resolve(rc->registry, call->callee_name, module_qn, imp_keys, - imp_vals, imp_count); + res = cbm_registry_resolve_lang(rc->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } atomic_fetch_add_explicit(&rc->time_ns_rc_resolve, extract_now_ns() - _rc_t0, memory_order_relaxed); @@ -3101,7 +3102,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB .strategy = "service_pattern"}; emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false); + imp_count, false, lang); continue; } } @@ -3118,7 +3119,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * self-call, so it keeps only the route/service edges. */ emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, rust_external); + imp_count, rust_external, lang); } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { /* Native `fetch()` (#856): only the global API once resolution * has failed to find a local/imported `fetch`. Call the low-level @@ -3170,7 +3171,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB if (url_or_topic) { emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false); + false, lang); ws->calls_resolved++; } } @@ -3179,7 +3180,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call); + drop_plain_call, lang); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; @@ -3245,8 +3246,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(rc->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(rc->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3295,7 +3296,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve throws/raises for one file. */ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *rel, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; if (!thr->exception_name || !thr->enclosing_func_qn) { @@ -3310,8 +3312,8 @@ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(rc->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + rc->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3337,8 +3339,8 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi if (!src) { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, rw->var_name, module_qn, imp_keys, - imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3365,12 +3367,14 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi /* Resolve base_classes → INHERITS edges for one definition. */ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->base_classes) { return; } for (int b = 0; def->base_classes[b]; b++) { - const char *bqn = resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic); + const char *bqn = + resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic, lang); if (!bqn) { continue; } @@ -3389,7 +3393,8 @@ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve decorators → DECORATES edges for one definition. */ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->decorators) { return; } @@ -3399,14 +3404,14 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws if (fn[0] == '\0') { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, fn, mq, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, fn, mq, ik, iv, ic, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(fn, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but * declared with an `Attribute` suffix (`class LogAttribute`). */ char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", fn); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(rc->registry, with_suffix, mq, ik, iv, ic); + res = cbm_registry_resolve_lang(rc->registry, with_suffix, mq, ik, iv, ic, lang); } } const cbm_gbuf_node_t *dn = NULL; @@ -3459,7 +3464,8 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws /* Resolve INHERITS + DECORATES + IMPLEMENTS for one file. */ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int d = 0; d < result->defs.count; d++) { CBMDefinition *def = &result->defs.items[d]; if (!def->qualified_name) { @@ -3469,8 +3475,8 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (!node) { continue; } - resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); - resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); + resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); } for (int t = 0; t < result->impl_traits.count; t++) { CBMImplTrait *it = &result->impl_traits.items[t]; @@ -3478,9 +3484,9 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *tqn = resolve_as_class(rc->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); const char *sqn = resolve_as_class(rc->registry, it->struct_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); if (!tqn || !sqn) { continue; } @@ -3763,7 +3769,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { (strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 || strcmp(result->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 || strcmp(result->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 || - strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0)) { + strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0)) { rc->lsp_failures[file_idx] = result->error_msg; ws->errors++; } @@ -3816,7 +3823,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── THROWS / RAISES ───────────────────────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_throws, extract_now_ns() - _ph_t0, memory_order_relaxed); @@ -3827,7 +3834,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, memory_order_relaxed); diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab00831..434ff769fd 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -177,9 +177,9 @@ static void free_import_map(const char **keys, const char **vals, int count) { /* Resolve a class/type name through the registry. Returns borrowed QN or NULL. */ static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -546,14 +546,14 @@ static void synth_decorator_qn(const char *func_name, char *out, size_t outsz) { /* Resolve one decorator and create DECORATES edge. */ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *node, const char *decorator, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, int *count) { + const char **imp_vals, int imp_count, int *count, CBMLanguage lang) { char func_name[CBM_SZ_256]; extract_decorator_func(decorator, func_name, sizeof(func_name)); if (func_name[0] == '\0') { return; } - cbm_resolution_t res = - cbm_registry_resolve(ctx->registry, func_name, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(func_name, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but declared * with the conventional `Attribute` suffix (`class LogAttribute`). Retry @@ -561,8 +561,8 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", func_name); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(ctx->registry, with_suffix, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, with_suffix, module_qn, imp_keys, + imp_vals, imp_count, lang); } } const cbm_gbuf_node_t *dec = NULL; @@ -613,7 +613,7 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, int *inherits_count, - int *decorates_count) { + int *decorates_count, CBMLanguage lang) { if (!def->qualified_name) { return; } @@ -624,7 +624,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->base_classes) { for (int b = 0; def->base_classes[b]; b++) { const char *base_qn = resolve_as_class(ctx->registry, def->base_classes[b], module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, lang); if (!base_qn) { continue; } @@ -646,7 +646,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->decorators) { for (int dc = 0; def->decorators[dc]; dc++) { resolve_decorator(ctx, node, def->decorators[dc], module_qn, imp_keys, imp_vals, - imp_count, decorates_count); + imp_count, decorates_count, lang); } } } @@ -683,12 +683,12 @@ static int resolve_impl_traits(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res continue; } const char *trait_qn = resolve_as_class(ctx->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, CBM_LANG_RUST); if (!trait_qn) { continue; } const char *struct_qn = resolve_as_class(ctx->registry, it->struct_name, module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, CBM_LANG_RUST); if (!struct_qn) { continue; } @@ -737,7 +737,7 @@ int cbm_pipeline_pass_semantic(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *f /* ── INHERITS + DECORATES from definitions ──────────────── */ for (int d = 0; d < result->defs.count; d++) { sem_process_def_edges(ctx, &result->defs.items[d], module_qn, imp_keys, imp_vals, - imp_count, &inherits_count, &decorates_count); + imp_count, &inherits_count, &decorates_count, files[i].language); } /* ── IMPLEMENTS from impl_traits (Rust) ─────────────────── */ diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 9738f7198d..096130775e 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -200,8 +200,8 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(ctx->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(ctx->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(ctx->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -248,7 +248,7 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res /* Resolve THROWS/RAISES edges for one file's extracted throws. */ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { int resolved = 0; for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; @@ -262,8 +262,8 @@ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + ctx->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); const cbm_gbuf_node_t *tgt = NULL; if (res.qualified_name && res.qualified_name[0]) { @@ -295,8 +295,8 @@ static int resolve_rw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result continue; } - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, rw->var_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -379,8 +379,8 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil usage_resolved += resolve_usage_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); - throw_resolved += - resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count); + throw_resolved += resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, + imp_count, files[i].language); rw_resolved += resolve_rw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae98..397691ee3d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -21,6 +21,15 @@ #include #include +/* Internal explicit-language resolution; public raw/lineage APIs stay raw. */ +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pxc_test_import_probe_fail_after(int successful_allocations); +#endif + /* ── Shared pipeline constants ─────────────────────────────────── */ /* Per-file tree-sitter parse budget, in MICROSECONDS of this thread's CPU time diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 1f5335c10c..bb8d19bdd1 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -24,7 +24,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #define REG_HALF_PENALTY 0.5 #define DEFAULT_CONFIDENCE 0.5 -#include "pipeline/pipeline.h" +#include "pipeline_internal.h" #include "cbm.h" /* cbm_label_is_relation — the resolve-time relation veto */ #include "foundation/compat.h" /* CBM_TLS */ #include "foundation/hash_table.h" @@ -387,6 +387,7 @@ void cbm_registry_import_map_cache_end(void) { * eliminated. */ typedef struct { cbm_resolution_t res; + size_t scope_len; /* raw and Python-scoped requests may share one file cache */ } resolve_cache_entry_t; static CBM_TLS CBMHashTable *_resolve_cache = NULL; @@ -1099,22 +1100,45 @@ static cbm_resolution_t resolve_import_map(const cbm_registry_t *r, const char * return empty_result(); } -/* Strategy 2: Same-module match */ -static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, - const char *suffix, const char *module_qn) { +/* Exact same-module lookup without ever hashing a truncated candidate. The + * long path uses the canonical short-name bucket, not heuristic name limits. */ +static const char *same_module_candidate(const cbm_registry_t *r, const char *module_qn, + size_t scope_len, const char *name) { + if (!module_qn || !name) + return NULL; + size_t name_len = strlen(name); char candidate[CBM_SZ_512]; - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, callee_name); - const char *stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } - if (suffix && suffix[0]) { - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, suffix); - stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } + if (scope_len < sizeof(candidate) && name_len < sizeof(candidate) - scope_len - 1) { + memcpy(candidate, module_qn, scope_len); + candidate[scope_len] = '.'; + memcpy(candidate + scope_len + 1, name, name_len + 1); + return cbm_ht_get_key(r->exact, candidate); + } + qn_array_t *arr = (qn_array_t *)cbm_ht_get(r->by_name, simple_name(name)); + if (!arr) + return NULL; + for (int i = 0; i < arr->count; i++) { + const char *qn = arr->items[i]; + if (!qn) + continue; + size_t qn_len = strlen(qn); + if (qn_len > scope_len && qn_len - scope_len - 1 == name_len && + memcmp(qn, module_qn, scope_len) == 0 && qn[scope_len] == '.' && + memcmp(qn + scope_len + 1, name, name_len) == 0) + return qn; } + return NULL; +} + +/* Strategy 2: Same-module match. Scoring elsewhere retains the raw file QN. */ +static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, + const char *suffix, const char *module_qn, + size_t scope_len) { + const char *stored_key = same_module_candidate(r, module_qn, scope_len, callee_name); + if (!stored_key && suffix && suffix[0]) + stored_key = same_module_candidate(r, module_qn, scope_len, suffix); + if (stored_key) + return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; return empty_result(); } @@ -1360,7 +1384,8 @@ static cbm_resolution_t resolve_name_lookup(const cbm_registry_t *r, const char * cbm_registry_resolve owns the per-file cache). */ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { + const char **import_map_vals, int import_map_count, + size_t scope_len) { /* Split callee at the first path separator: "pkg.Func" → prefix="pkg", * suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever * separator appears first ("lib::square" → prefix="lib", suffix="square"). @@ -1393,7 +1418,7 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch resolve_import_map(r, prefix, suffix, import_map_keys, import_map_vals, import_map_count); if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 2: same module */ - res = resolve_same_module(r, callee_name, suffix, module_qn); + res = resolve_same_module(r, callee_name, suffix, module_qn, scope_len); } if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 3+4: name lookup */ @@ -1402,9 +1427,10 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch return res; } -cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { +static cbm_resolution_t registry_resolve_scoped(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + size_t scope_len) { if (!r || !callee_name) { return empty_result(); } @@ -1412,16 +1438,13 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Per-file cache: same callee_name in N call sites → 1 chain walk * + N-1 O(1) hash hits. module_qn is constant per file so the * cache key only needs callee_name. */ - if (_resolve_cache) { - resolve_cache_entry_t *cached = - (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); - if (cached) { - return cached->res; - } - } + resolve_cache_entry_t *cached = + _resolve_cache ? (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name) : NULL; + if (cached && cached->scope_len == scope_len) + return cached->res; cbm_resolution_t res = registry_resolve_chain(r, callee_name, module_qn, import_map_keys, - import_map_vals, import_map_count); + import_map_vals, import_map_count, scope_len); /* Data relations (Table/View) are lineage-only registry members: common * table names (users, orders, config) collide with code identifiers across @@ -1437,10 +1460,15 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Cache the result (including empty — caching the negative answer * is just as valuable; same name asks the same question). */ - if (_resolve_cache) { + if (cached) { + /* Same key, different effective scope: reuse its owned entry/key. */ + cached->res = res; + cached->scope_len = scope_len; + } else if (_resolve_cache) { resolve_cache_entry_t *e = (resolve_cache_entry_t *)malloc(sizeof(*e)); if (e) { e->res = res; + e->scope_len = scope_len; char *kdup = strdup(callee_name); if (kdup) { cbm_ht_set(_resolve_cache, kdup, e); @@ -1452,6 +1480,23 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle return res; } +cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, module_qn ? strlen(module_qn) : 0); +} + +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang) { + size_t scope_len = lang == CBM_LANG_PYTHON ? cbm_fqn_symbol_scope_len(module_qn) + : (module_qn ? strlen(module_qn) : 0); + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, scope_len); +} + cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count) { @@ -1464,7 +1509,7 @@ cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const cha * it would poison one variant with the other's semantics. SQL files hold * few distinct relation refs, so the chain walk stays cheap. */ return registry_resolve_chain(r, callee_name, module_qn, import_map_keys, import_map_vals, - import_map_count); + import_map_count, module_qn ? strlen(module_qn) : 0); } /* ── Fuzzy Resolve ──────────────────────────────────────────────── */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ba220bc84b..a7080137b9 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8530,6 +8530,352 @@ TEST(pipeline_python_module_callers_parallel) { return python_module_callers_pipeline_case(true); } +/* Python file identity keeps .__init__; its exported symbols do not. */ +static bool python_init_local_inheritance(cbm_store_t *store, const char *project, + const char *module) { + char child_qn[768], parent_qn[768], wrong_qn[768]; + int cn = snprintf(child_qn, sizeof(child_qn), "%s.%s.BooleanField", project, module); + int pn = snprintf(parent_qn, sizeof(parent_qn), "%s.%s.Field", project, module); + int wn = snprintf(wrong_qn, sizeof(wrong_qn), "%s.other.Field", project); + if (cn <= 0 || (size_t)cn >= sizeof(child_qn) || pn <= 0 || (size_t)pn >= sizeof(parent_qn) || + wn <= 0 || (size_t)wn >= sizeof(wrong_qn)) + return false; + cbm_node_t child = {0}, parent = {0}, wrong = {0}; + bool ok = cbm_store_find_node_by_qn(store, project, child_qn, &child) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, parent_qn, &parent) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, wrong_qn, &wrong) == CBM_STORE_OK && + child.id > 0 && parent.id > 0 && wrong.id > 0 && parent.id != wrong.id && + child.label && parent.label && wrong.label && strcmp(child.label, "Class") == 0 && + strcmp(parent.label, "Class") == 0 && strcmp(wrong.label, "Class") == 0; + cbm_edge_t *edges = NULL; + int count = 0, local = 0, foreign = 0; + if (ok) { + ok = cbm_store_find_edges_by_source_type(store, child.id, "INHERITS", &edges, &count) == + CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + local += edges[i].target_id == parent.id; + foreign += edges[i].target_id == wrong.id; + } + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&child); + cbm_node_free_fields(&parent); + cbm_node_free_fields(&wrong); + return ok && local == 1 && foreign == 0; +} + +static int python_init_scope_pipeline_case(bool parallel, const char *failure_stage) { + static const char body[] = "from util import normalize\n" + "from other import Field as OtherField\n" + "def helper(value):\n return normalize(value)\n" + "class Field:\n" + " def clean(self, value):\n return helper(value)\n" + " def validate(self, value):\n return self.clean(value)\n" + "class BooleanField(Field):\n" + " def to_python(self, value):\n" + " self.validate(value)\n return helper(value)\n" + "def build(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const char consumer[] = "from pkg import BooleanField, helper\n" + "def consume(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } local_edges[] = { + {"Field.clean", "Method", "helper", "Function"}, + {"Field.validate", "Method", "Field.clean", "Method"}, + {"BooleanField.to_python", "Method", "Field.validate", "Method"}, + {"BooleanField.to_python", "Method", "helper", "Function"}, + {"build", "Function", "BooleanField.to_python", "Method"}, + {"build", "Function", "helper", "Function"}, + }; + char root[512], path[768]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-init-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python package fixture"); + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file( + root, "pkg/__init__.py", + failure_stage ? "class S:\n def m(self):\n return 1\n" + "def run():\n s = S()\n return s.m()\n" + : body); + } + if (written && !failure_stage) { + written = python_module_write_fixture_file(root, "plain.py", body) && + python_module_write_fixture_file(root, "consumer.py", consumer) && + python_module_write_fixture_file(root, "util.py", + "def normalize(value):\n return value\n") && + python_module_write_fixture_file(root, "other.py", "class Field:\n pass\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "init_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "# inert parallel routing file\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python package fixture"); + } + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + "CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"}; + char *saved[3] = {0}; + bool saved_ok = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + for (int i = 0; i < 3; i++) + free(saved[i]); + th_rmtree(root); + FAIL("failed to save Python package fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= failure_stage ? cbm_setenv(keys[2], failure_stage, 1) : cbm_unsetenv(keys[2]); + n = snprintf(path, sizeof(path), "%s/package.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0, scope_errors = 0; + bool precise = true; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + for (int i = 0; i < error_count; i++) { + bool is_package = errors[i].path && strcmp(errors[i].path, "pkg/__init__.py") == 0; + if (failure_stage && is_package) { + scope_errors++; + const char *phase = strcmp(failure_stage, "raw") == 0 ? "extract" : "lsp_skipped"; + precise = precise && errors[i].phase && strcmp(errors[i].phase, phase) == 0 && + errors[i].reason && strcmp(errors[i].reason, CBM_PY_LSP_SCOPE_ERROR) == 0; + } else { + precise = false; + } + } + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + int file_count = 0; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (failure_stage ? 1 : 5) + (parallel ? 52 : 0); + cbm_store_free_nodes(files, file_count); + if (project && !failure_stage) { + const char *modules[] = {"plain", "pkg"}; + for (int m = 0; m < 2; m++) { + for (size_t i = 0; i < sizeof(local_edges) / sizeof(local_edges[0]); i++) { + char caller[128], target[128]; + snprintf(caller, sizeof(caller), "%s.%s", modules[m], local_edges[i].caller); + snprintf(target, sizeof(target), "%s.%s", modules[m], local_edges[i].target); + bool edge = python_module_exact_lsp_edge(store, project, caller, + local_edges[i].caller_label, target, + local_edges[i].target_label); + correct = correct && edge; + } + char caller[128]; + snprintf(caller, sizeof(caller), "%s.helper", modules[m]); + bool imported = python_module_exact_lsp_edge(store, project, caller, "Function", + "util.normalize", "Function"); + bool inherits = python_init_local_inheritance(store, project, modules[m]); + correct = correct && imported && inherits; + } + bool method = + python_module_exact_lsp_edge(store, project, "consumer.consume", "Function", + "pkg.BooleanField.to_python", "Method"); + bool function = python_module_exact_lsp_edge(store, project, "consumer.consume", + "Function", "pkg.helper", "Function"); + correct = correct && method && function; + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, failure_stage ? 1 : 0); + ASSERT_EQ(scope_errors, failure_stage ? 1 : 0); + ASSERT_TRUE(precise && correct); + PASS(); +} + +TEST(pipeline_python_init_scope_sequential) { + return python_init_scope_pipeline_case(false, NULL); +} +TEST(pipeline_python_init_scope_parallel) { + return python_init_scope_pipeline_case(true, NULL); +} +TEST(pipeline_python_init_raw_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "raw"); +} +TEST(pipeline_python_init_raw_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "raw"); +} +TEST(pipeline_python_init_cross_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "cross"); +} +TEST(pipeline_python_init_cross_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "cross"); +} +TEST(pipeline_python_init_register_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "register"); +} +TEST(pipeline_python_init_register_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "register"); +} + +/* Compare identities, not just counts; a duplicate import must stop at the + * same nearest eligible entry even when it contributes no new selections. */ +static bool python_init_filter_is(CBMLSPDef *defs, int def_count, const char *const *imports, + int import_count, const char *const *expected, int count) { + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, def_count); + int got_count = -1; + bool succeeded = false; + CBMLSPDef *got = + cbm_pxc_filter_defs_for_file(index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", imports, + import_count, &got_count, &succeeded); + bool correct = index && succeeded && got_count == count && (count == 0 || got); + for (int i = 0; correct && i < count; i++) { + int hits = 0; + for (int j = 0; j < got_count; j++) + hits += got[j].qualified_name && strcmp(got[j].qualified_name, expected[i]) == 0; + correct = hits == 1; + } + free(got); + cbm_pxc_free_module_def_index(index); + return correct; +} + +TEST(pipeline_python_init_import_filter_precedence) { + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Foreign", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.Parent", + .def_module_qn = "test.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.foreign.Only", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.unknown.Only", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1}, + {.qualified_name = "test.plain.Exact", + .def_module_qn = "test.plain", + .lang = CBM_LANG_JAVA}, + {.qualified_name = "test.plain.Package", + .def_module_qn = "test.plain.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.nested.Item", + .def_module_qn = "test.pkg.nested.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Item", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + }; + const char *pkg[] = {"test.pkg", "test.pkg.Store", "test.pkg.Store"}; + const char *want_pkg[] = {"test.consumer.use", "test.pkg.Store"}; + const char *foreign[] = {"test.foreign.Only", "test.unknown.Only"}; + const char *want_parent[] = {"test.consumer.use", "test.Parent"}; + const char *normal[] = {"test.plain.Exact", "test.plain.Exact"}; + const char *want_normal[] = {"test.consumer.use", "test.plain.Exact"}; + const char *nested[] = {"test.pkg.nested.Item", "test.pkg.nested.Item"}; + const char *want_nested[] = {"test.consumer.use", "test.pkg.nested.Item"}; + const char *submodule[] = {"test.pkg.sub.Item"}; + const char *want_submodule[] = {"test.consumer.use", "test.pkg.sub.Item"}; + int n = (int)(sizeof(defs) / sizeof(defs[0])); + bool pkg_ok = python_init_filter_is(defs, n, pkg, 3, want_pkg, 2); + bool foreign_ok = python_init_filter_is(defs, n, foreign, 2, want_parent, 2); + bool normal_ok = python_init_filter_is(defs, n, normal, 2, want_normal, 2); + bool nested_ok = python_init_filter_is(defs, n, nested, 2, want_nested, 2); + bool submodule_ok = python_init_filter_is(defs, n, submodule, 1, want_submodule, 2); + ASSERT_TRUE(pkg_ok && foreign_ok && normal_ok && nested_ok && submodule_ok); + PASS(); +} + +TEST(pipeline_python_init_import_probe_failure_falls_back) { + const char *source = "from pkg import Store\n" + "def use():\n store = Store()\n return store.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .short_name = "use", + .label = "Function", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *keys[] = {"Store"}; + const char *values[] = {"test.pkg.Store"}; + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, 3); + int count = -1; + bool succeeded = true; + cbm_pxc_test_import_probe_fail_after(0); + CBMLSPDef *filtered = cbm_pxc_filter_defs_for_file( + index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", values, 1, &count, &succeeded); + cbm_pxc_test_import_probe_fail_after(-1); + bool failed_precisely = index && !filtered && count == 0 && !succeeded; + free(filtered); + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_pxc_test_import_probe_fail_after(0); + cbm_pxc_dispatch_file(CBM_LANG_PYTHON, &result, source, (int)strlen(source), "consumer.py", + "test.consumer", NULL, index, defs, 3, keys, values, 1, NULL, NULL); + cbm_pxc_test_import_probe_fail_after(-1); + const char *site = strstr(source, "store.get()"); + int exact = 0; + for (int i = 0; i < result.resolved_calls.count; i++) { + const CBMResolvedCall *call = &result.resolved_calls.items[i]; + if (call->caller_qn && strcmp(call->caller_qn, "test.consumer.use") == 0 && + call->callee_qn && strcmp(call->callee_qn, "test.pkg.Store.get") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && call->strategy && + strncmp(call->strategy, "lsp_", 4) == 0 && site && + call->site_start_byte == (uint32_t)(site - source) && + call->site_end_byte == (uint32_t)(site - source + strlen("store.get()"))) + exact++; + } + bool resolved = !result.has_error && !result.lsp_skipped && exact == 1; + cbm_arena_destroy(&result.arena); + cbm_pxc_free_module_def_index(index); + ASSERT_TRUE(failed_precisely && resolved); + PASS(); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16776,6 +17122,16 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { SUITE(pipeline) { RUN_TEST(pipeline_python_module_callers_sequential); RUN_TEST(pipeline_python_module_callers_parallel); + RUN_TEST(pipeline_python_init_scope_sequential); + RUN_TEST(pipeline_python_init_scope_parallel); + RUN_TEST(pipeline_python_init_raw_scope_failure_sequential); + RUN_TEST(pipeline_python_init_raw_scope_failure_parallel); + RUN_TEST(pipeline_python_init_cross_scope_failure_sequential); + RUN_TEST(pipeline_python_init_cross_scope_failure_parallel); + RUN_TEST(pipeline_python_init_register_scope_failure_sequential); + RUN_TEST(pipeline_python_init_register_scope_failure_parallel); + RUN_TEST(pipeline_python_init_import_filter_precedence); + RUN_TEST(pipeline_python_init_import_probe_failure_falls_back); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index adc84913b3..b16f5f87a8 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -2913,7 +2913,403 @@ TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites) { PASS(); } +/* Package symbols use their parent scope while Module callers retain the file. */ +TEST(pylsp_init_scope_context_and_helper) { + static const struct { + const char *raw; + const char *scope; + } cases[] = { + {"test.__init__", "test"}, + {"test.pkg.__init__", "test.pkg"}, + {"test.pkg.nested.__init__", "test.pkg.nested"}, + {"test.plain", "test.plain"}, + {"test.x__init__", "test.x__init__"}, + {"__init__", "__init__"}, + {"test.index", "test.index"}, + {NULL, NULL}, + }; + bool correct = true; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry registry; + cbm_registry_init(®istry, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, "pass\n", 5, ®istry, cases[i].raw, &out); + correct = + correct && ctx.eval_failure == CBM_LSP_COMPLETE && ctx.current_scope && + ctx.file_module_qn == cases[i].raw && + cbm_fqn_symbol_scope_len(cases[i].raw) == (cases[i].scope ? strlen(cases[i].scope) : 0); + correct = + correct && (cases[i].scope ? ctx.module_qn && strcmp(ctx.module_qn, cases[i].scope) == 0 + : ctx.module_qn == NULL); + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations) { + static const char source[] = + "class Store:\n def get(self):\n return 1\n" + "def identity(value: Store) -> Store:\n return value\n" + "def quoted(value: 'Store') -> 'Store':\n return value\n" + "def wrapped(values: list[Store]) -> Store:\n return values[0]\n" + "def run(first: Store, second: 'Store', values: list[Store]):\n" + " first.get()\n second.get()\n values[0].get()\n" + " identity(first).get()\n quoted(second).get()\n" + " wrapped(values).get()\n" + "class User:\n def use(self, item: Store):\n return item.get()\n" + "top = Store()\ntop.get()\n"; + static const char *paths[] = {"plain.py", "__init__.py", "pkg/nested/__init__.py"}; + static const char *scopes[] = {"test.plain", "test", "test.pkg.nested"}; + static const char *modules[] = {"test.plain", "test.__init__", "test.pkg.nested.__init__"}; + static const char *sites[] = {"first.get()", "second.get()", + "values[0].get()", "identity(first).get()", + "quoted(second).get()", "wrapped(values).get()"}; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[mode], 0, NULL, NULL); + bool extracted = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + r->module_qn && strcmp(r->module_qn, modules[mode]) == 0; + correct = correct && extracted; + if (extracted) { + char caller[128], target[128], method[128]; + snprintf(caller, sizeof(caller), "%s.run", scopes[mode]); + snprintf(target, sizeof(target), "%s.Store.get", scopes[mode]); + snprintf(method, sizeof(method), "%s.User.use", scopes[mode]); + correct = correct && pylsp_module_has_def(r, caller) && + pylsp_module_has_def(r, target) && pylsp_module_has_def(r, method); + for (size_t j = 0; j < sizeof(sites) / sizeof(sites[0]); j++) { + bool joined = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + sites[j], caller, target); + correct = correct && joined; + } + bool method_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "item.get()", method, target); + bool module_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "top.get()", modules[mode], target); + correct = correct && method_join && module_join; + } + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +static bool pylsp_init_named_type(const CBMType *type, const char *qn) { + return type && type->kind == CBM_TYPE_NAMED && type->data.named.qualified_name && + strcmp(type->data.named.qualified_name, qn) == 0; +} + +static bool pylsp_init_signature(const CBMTypeRegistry *registry, const char *function, + const char *expected_type) { + const CBMRegisteredFunc *f = cbm_registry_lookup_func(registry, function); + const CBMType *sig = f ? f->signature : NULL; + if (!sig || sig->kind != CBM_TYPE_FUNC || !sig->data.func.param_types) + return false; + const CBMType **params = sig->data.func.param_types; + if (!pylsp_init_named_type(params[0], expected_type) || + !pylsp_init_named_type(params[1], expected_type) || !params[2] || params[3]) + return false; + const CBMType *container = params[2]; + return container->kind == CBM_TYPE_TEMPLATE && container->data.template_type.template_name && + strcmp(container->data.template_type.template_name, "list") == 0 && + container->data.template_type.arg_count == 1 && + container->data.template_type.template_args && + pylsp_init_named_type(container->data.template_type.template_args[0], expected_type); +} + +TEST(pylsp_init_scope_cross_signatures_keep_definition_language) { + const char *params[] = {"Token", "'Token'", "list[Token]"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.foreign.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.unknown.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1, + .signature_param_types = params, + .signature_param_count = 3}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry fallback; + cbm_registry_init(&fallback, &arena); + bool registered = cbm_py_lsp_test_register_defs(&arena, &fallback, defs, 3); + bool correct = registered && + pylsp_init_signature(&fallback, "test.pkg.call", "test.pkg.Token") && + pylsp_init_signature(&fallback, "test.foreign.__init__.call", + "test.foreign.__init__.Token") && + pylsp_init_signature(&fallback, "test.unknown.__init__.call", + "test.unknown.__init__.Token"); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && base && base->read_only && + pylsp_init_signature(base, "test.pkg.call", "test.pkg.Token") && + !cbm_registry_lookup_func(base, "test.foreign.__init__.call") && + !cbm_registry_lookup_func(base, "test.unknown.__init__.call"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_external_classes_and_cross_exact_joins) { + const char *source = "from pkg.sub import Widget as Imported\n" + "def use():\n store = Store()\n store.get()\n" + " imported = Imported()\n imported.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget", + .short_name = "Widget", + .label = "Class", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.sub", + .receiver_type = "test.pkg.sub.Widget", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Outer.Nested", + .short_name = "Nested", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + }; + const char *names[] = {"Imported"}; + const char *qns[] = {"test.pkg.sub.Widget"}; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + TSParser *parser = ts_parser_new(); + if (parser) + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + parser ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && tree && + pylsp_module_has_def(r, "test.pkg.use"); + for (int prebuilt = 0; correct && prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 5); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &overlay, "test.pkg.__init__", &out); + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool globals = base && ctx.eval_failure == CBM_LSP_COMPLETE && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "Store"), + "test.pkg.Store") && + !cbm_scope_lookup_local(ctx.current_scope, "Widget") && + !cbm_scope_lookup_local(ctx.current_scope, "Nested"); + char module_input[] = "test.pkg.__init__"; + CBMLSPStatus status = + prebuilt + ? cbm_run_py_lsp_cross_with_registry_status(&arena, source, (int)strlen(source), + module_input, &overlay, names, qns, 1, + tree, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), module_input, + defs, 5, names, qns, 1, tree, &out, NULL); + memset(module_input, 'x', sizeof(module_input) - 1); + bool store = pylsp_module_site_joins(&r->calls, &out, source, "store.get()", "test.pkg.use", + "test.pkg.Store.get"); + bool imported = pylsp_module_site_joins(&r->calls, &out, source, "imported.get()", + "test.pkg.use", "test.pkg.sub.Widget.get"); + correct = correct && globals && status == CBM_LSP_COMPLETE && store && imported; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers) { + const char *source = "def run():\n value = Store()\n return value.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + cbm_py_lsp_test_scope_fail_after(0); + CBMFileResult *raw = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && + raw->resolved_calls.count == 0 && pylsp_module_has_def(raw, "test.pkg.run"); + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 2); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCall prior = {.caller_qn = "completed.caller", + .callee_qn = "completed.target", + .strategy = "prior.strategy", + .reason = "prior.reason", + .confidence = 0.125f, + .kind = CBM_RESOLVED_INVOCATION, + .site_start_byte = 7, + .site_end_byte = 14}; + unsigned char saved[sizeof(prior)]; + memcpy(saved, &prior, sizeof(prior)); + CBMCall previous_synthetic = {.callee_name = "completed.synthetic"}; + CBMCallArray synthetic = {.items = &previous_synthetic, .count = 1, .cap = 1}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + cbm_py_lsp_test_scope_fail_after(mode == 2 ? 1 : 0); + /* mode 2 gets past context normalization, then fails signature scope. */ + CBMLSPStatus status = mode == 1 + ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), "test.pkg.__init__", + &overlay, NULL, NULL, 0, NULL, &out, &synthetic) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + "test.pkg.__init__", defs, 2, NULL, + NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && base && status == CBM_LSP_SCOPE_FAILED && out.count == 1 && + out.items == &prior && memcmp(saved, &prior, sizeof(prior)) == 0 && + synthetic.count == 1 && synthetic.items == &previous_synthetic && + strcmp(previous_synthetic.callee_name, "completed.synthetic") == 0; + cbm_py_lsp_test_scope_fail_after(0); + bool complete = + mode == 1 + ? cbm_run_py_lsp_cross_with_registry(&arena, source, (int)strlen(source), + "test.pkg.__init__", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.pkg.__init__", + defs, 2, NULL, NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = + correct && !complete && out.count == 1 && memcmp(saved, &prior, sizeof(prior)) == 0; + CBMBatchPyLSPFile file = {.source = source, + .source_len = (int)strlen(source), + .module_qn = "test.pkg.__init__", + .defs = defs, + .def_count = 2}; + CBMResolvedCallArray batch = {0}; + cbm_py_lsp_test_scope_fail_after(0); + bool batch_complete = cbm_batch_py_lsp_cross(&arena, &file, 1, &batch); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && !batch_complete && batch.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback) { + const char *source = "def use():\n obj = Item()\n return obj.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.plain.Item", + .short_name = "Item", + .label = "Class", + .def_module_qn = "test.plain", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.plain.Item.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.plain", + .receiver_type = "test.plain.Item", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.unused.work", + .short_name = "work", + .label = "Function", + .def_module_qn = "test.unused.__init__", + .lang = CBM_LANG_PYTHON}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + char *sentinel = cbm_arena_strdup(&shared, "caller-owned shared data"); + cbm_py_lsp_test_scope_fail_after(0); + CBMTypeRegistry *failed = cbm_py_build_cross_registry(&shared, defs, 3); + CBMResolvedCallArray out = {0}; + CBMLSPStatus fallback = + cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), "test.plain", defs, 2, + NULL, NULL, 0, NULL, &out, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + bool correct = !failed && sentinel && strcmp(sentinel, "caller-owned shared data") == 0 && + fallback == CBM_LSP_COMPLETE; + CBMFileResult *extracted = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "test", "plain.py", 0, NULL, NULL); + correct = correct && extracted && !extracted->has_error && !extracted->parse_incomplete && + pylsp_module_has_def(extracted, "test.plain.use") && + pylsp_module_site_joins(&extracted->calls, &out, source, "obj.get()", + "test.plain.use", "test.plain.Item.get"); + CBMTypeRegistry *fresh = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && fresh && fresh->read_only && + cbm_registry_lookup_func(fresh, "test.unused.work") && + strcmp(sentinel, "caller-owned shared data") == 0; + if (extracted) + cbm_free_result(extracted); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_init_scope_context_and_helper); + RUN_TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations); + RUN_TEST(pylsp_init_scope_cross_signatures_keep_definition_language); + RUN_TEST(pylsp_init_scope_external_classes_and_cross_exact_joins); + RUN_TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers); + RUN_TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback); + RUN_TEST(pylsp_module_callers_join_exact_raw_occurrences); RUN_TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair); RUN_TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites); diff --git a/tests/test_registry.c b/tests/test_registry.c index 278e97d9b0..4d305356c4 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1288,7 +1288,277 @@ TEST(resolve_import_map_alias_with_suffix_hits_method) { PASS(); } +/* Python package scope is an explicit resolver input, not a property inferred + * from a QN shared by other languages. Keep these fixtures separate from the + * raw public-API tests above. */ +static bool registry_scope_match(cbm_resolution_t result, const char *qn, const char *strategy) { + return qn && strategy && result.qualified_name && result.strategy && + strcmp(result.qualified_name, qn) == 0 && strcmp(result.strategy, strategy) == 0 && + result.candidate_count == 1; +} + +TEST(python_symbol_scope_len_preserves_file_identity) { + const char *inputs[] = {"proj.pkg.__init__", + "proj.__init__", + "proj.pkg.__init__.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + NULL}; + const char *scopes[] = {"proj.pkg", + "proj", + "proj.pkg.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + ""}; + bool lengths = true; + for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); i++) { + lengths = cbm_fqn_symbol_scope_len(inputs[i]) == strlen(scopes[i]) && lengths; + } + char *module = cbm_pipeline_fqn_module("proj", "pkg/__init__.py"); + char *symbol = cbm_pipeline_fqn_compute("proj", "pkg/__init__.py", "Field"); + bool identities = module && symbol && strcmp(module, "proj.pkg.__init__") == 0 && + strcmp(symbol, "proj.pkg.Field") == 0 && + cbm_fqn_symbol_scope_len(module) == strlen("proj.pkg"); + free(module); + free(symbol); + ASSERT_TRUE(lengths && identities); + PASS(); +} + +TEST(registry_python_package_scope_is_explicit) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.forms.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.__init__.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.plain.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.index.Field", "Class"); + const char *keys[] = {"forms"}; + const char *vals[] = {"proj.forms"}; + const char *module = "proj.db.fields.__init__"; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t package = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, CBM_LANG_PYTHON); + bool python = registry_scope_match(package, "proj.db.fields.Field", "same_module"); + cbm_resolution_t plain = + cbm_registry_resolve_lang(r, "Field", "proj.db.plain", keys, vals, 1, CBM_LANG_PYTHON); + bool normal = registry_scope_match(plain, "proj.db.plain.Field", "same_module"); + cbm_resolution_t raw = cbm_registry_resolve(r, "Field", module, keys, vals, 1); + bool preserved = registry_scope_match(raw, "proj.db.fields.__init__.Field", "same_module"); + const CBMLanguage raw_languages[] = {CBM_LANG_JAVA, CBM_LANG_GO, CBM_LANG_JAVASCRIPT, + (CBMLanguage)-1, CBM_LANG_COUNT}; + for (size_t i = 0; i < sizeof(raw_languages) / sizeof(raw_languages[0]); i++) { + cbm_resolution_t result = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, raw_languages[i]); + preserved = registry_scope_match(result, raw.qualified_name, "same_module") && + result.confidence == raw.confidence && preserved; + } + cbm_resolution_t index = + cbm_registry_resolve_lang(r, "Field", "proj.db.index", keys, vals, 1, CBM_LANG_JAVASCRIPT); + bool index_raw = registry_scope_match(index, "proj.db.index.Field", "same_module"); + cbm_registry_free(r); + ASSERT_TRUE(seeded && python && normal && preserved && index_raw); + PASS(); +} + +TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.vendor.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.__init__.Field", "Class"); + cbm_registry_add(r, "choose", "proj.pkg.__init__.child.choose", "Function"); + cbm_registry_add(r, "choose", "proj.pkg.peer.choose", "Function"); + const char *keys[] = {"Field"}; + const char *vals[] = {"proj.vendor.Field"}; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t imported = + cbm_registry_resolve_lang(r, "Field", "proj.pkg.__init__", keys, vals, 1, CBM_LANG_PYTHON); + bool precedence = registry_scope_match(imported, "proj.vendor.Field", "import_map"); + cbm_resolution_t raw = cbm_registry_resolve(r, "choose", "proj.pkg.__init__", NULL, NULL, 0); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, "choose", "proj.pkg.__init__", NULL, NULL, 0, CBM_LANG_PYTHON); + bool scoring = + raw.qualified_name && python.qualified_name && raw.strategy && python.strategy && + strcmp(raw.qualified_name, "proj.pkg.__init__.child.choose") == 0 && + strcmp(python.qualified_name, raw.qualified_name) == 0 && + strcmp(raw.strategy, "suffix_match") == 0 && strcmp(python.strategy, raw.strategy) == 0 && + python.confidence == raw.confidence && python.candidate_count == raw.candidate_count; + cbm_registry_free(r); + ASSERT_TRUE(seeded && precedence && scoring); + PASS(); +} + +TEST(registry_python_scope_cache_modes_and_raw_lineage) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Entry", "proj.pkg.Entry", "Function"); + cbm_registry_add(r, "Entry", "proj.pkg.__init__.Entry", "Function"); + cbm_registry_add(r, "Reject", "proj.pkg.Reject", "Table"); + cbm_registry_add(r, "Reject", "proj.pkg.__init__.Reject", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.Accept", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.__init__.Accept", "Table"); + cbm_registry_add(r, "Relation", "proj.pkg.Relation", "View"); + cbm_registry_add(r, "Relation", "proj.pkg.__init__.Relation", "Table"); + bool seeded = cbm_registry_size(r) == 8; + bool correct = true; + /* Each cache lifetime has one fixed module/import map. Alternate modes in + * both starting orders; repeated same-mode requests also exercise hits. */ + for (int first_python = 0; first_python < 2; first_python++) { + cbm_registry_resolve_cache_begin(8); + for (int turn = 0; turn < 16; turn++) { + bool python = ((turn / 2 + first_python) % 2) != 0; + const char *names[] = {"Entry", "Reject", "Accept", "Relation", "Missing"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + cbm_resolution_t result = + python ? cbm_registry_resolve_lang(r, names[i], "proj.pkg.__init__", NULL, NULL, + 0, CBM_LANG_PYTHON) + : cbm_registry_resolve(r, names[i], "proj.pkg.__init__", NULL, NULL, 0); + const char *expected = NULL; + if (i == 0) { + expected = python ? "proj.pkg.Entry" : "proj.pkg.__init__.Entry"; + } else if (i == 1 && !python) { + expected = "proj.pkg.__init__.Reject"; + } else if (i == 2 && python) { + expected = "proj.pkg.Accept"; + } + correct = + (expected ? registry_scope_match(result, expected, "same_module") + : result.qualified_name == NULL && result.candidate_count == 0) && + correct; + } + /* The cached relation veto must never poison the uncached SQL API, + * nor may the SQL result bypass the next default cached veto. */ + cbm_resolution_t lineage = + cbm_registry_resolve_lineage(r, "Relation", "proj.pkg.__init__", NULL, NULL, 0); + correct = registry_scope_match(lineage, "proj.pkg.__init__.Relation", "same_module") && + correct; + cbm_resolution_t veto = cbm_registry_resolve_lang(r, "Relation", "proj.pkg.__init__", + NULL, NULL, 0, CBM_LANG_PYTHON); + correct = veto.qualified_name == NULL && veto.candidate_count == 0 && correct; + } + cbm_registry_resolve_cache_end(); + } + cbm_registry_free(r); + ASSERT_TRUE(seeded && correct); + PASS(); +} + +static const char *registry_scope_canonical(cbm_registry_t *r, const char *name, const char *qn) { + const char **candidates = NULL; + int count = 0; + cbm_registry_find_by_name(r, name, &candidates, &count); + for (int i = 0; i < count; i++) { + if (strcmp(candidates[i], qn) == 0) { + return candidates[i]; + } + } + return NULL; +} + +TEST(registry_python_scope_complete_candidate_boundaries) { + const size_t lengths[] = {511, 512, 513, 1536}; + bool complete = true; + for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { + cbm_registry_t *r = cbm_registry_new(); + if (!r) { + complete = false; + break; + } + char scope[2048]; + char module[2048]; + char target[2048]; + const char *name = "LongTarget"; + size_t scope_len = lengths[i] - strlen(name) - 1; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'a', scope_len - 5); + scope[scope_len] = '\0'; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + int tn = snprintf(target, sizeof(target), "%s.%s", scope, name); + bool shape = mn > 0 && (size_t)mn < sizeof(module) && tn > 0 && (size_t)tn == lengths[i] && + (size_t)tn < sizeof(target); + cbm_registry_add(r, name, target, "Function"); + if (lengths[i] > 511) { + char truncated[512]; + memcpy(truncated, target, sizeof(truncated) - 1); + truncated[sizeof(truncated) - 1] = '\0'; + cbm_registry_add(r, "truncated_decoy", truncated, "Function"); + } + const char *canonical = registry_scope_canonical(r, name, target); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, name, module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t normal = cbm_registry_resolve(r, name, scope, NULL, NULL, 0); + complete = shape && canonical && python.qualified_name == canonical && + normal.qualified_name == canonical && + registry_scope_match(python, target, "same_module") && + registry_scope_match(normal, target, "same_module") && complete; + cbm_registry_free(r); + } + ASSERT_TRUE(complete); + PASS(); +} + +TEST(registry_python_scope_qualified_terminal_bucket_uncapped) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + /* The exact targets are inserted after the heuristic limit. A long exact + * comparison must use the canonical terminal bucket, not "Owner.perform" + * as the bucket key, and must not stop at 256 candidates. */ + bool shape = true; + for (int i = 0; i < 300; i++) { + char qn[80]; + int n = snprintf(qn, sizeof(qn), "proj.noise%d.perform", i); + shape = n > 0 && (size_t)n < sizeof(qn) && shape; + cbm_registry_add(r, "perform", qn, "Method"); + } + char scope[1024]; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'b', 900); + scope[905] = '\0'; + char module[1024]; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + shape = mn > 0 && (size_t)mn < sizeof(module) && shape; + const char *names[] = {"Owner.perform", "Owner::perform"}; + bool resolved = true; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char target[1024]; + char prefixed[80]; + int tn = snprintf(target, sizeof(target), "%s.%s", scope, names[i]); + int pn = snprintf(prefixed, sizeof(prefixed), "receiver.%s", names[i]); + shape = tn > 0 && (size_t)tn < sizeof(target) && pn > 0 && (size_t)pn < sizeof(prefixed) && + shape; + cbm_registry_add(r, "perform", target, "Method"); + const char *canonical = registry_scope_canonical(r, "perform", target); + cbm_resolution_t direct = + cbm_registry_resolve_lang(r, names[i], module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t suffix = + cbm_registry_resolve_lang(r, prefixed, module, NULL, NULL, 0, CBM_LANG_PYTHON); + resolved = canonical && direct.qualified_name == canonical && + suffix.qualified_name == canonical && + registry_scope_match(direct, target, "same_module") && + registry_scope_match(suffix, target, "same_module") && resolved; + } + bool seeded = cbm_registry_size(r) == 302; + cbm_registry_free(r); + ASSERT_TRUE(shape && seeded && resolved); + PASS(); +} + SUITE(registry) { + RUN_TEST(python_symbol_scope_len_preserves_file_identity); + RUN_TEST(registry_python_package_scope_is_explicit); + RUN_TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring); + RUN_TEST(registry_python_scope_cache_modes_and_raw_lineage); + RUN_TEST(registry_python_scope_complete_candidate_boundaries); + RUN_TEST(registry_python_scope_qualified_terminal_bucket_uncapped); /* FQN */ RUN_TEST(fqn_simple); RUN_TEST(fqn_no_name); From 71337a2853c1aef198dc0b882565a817d3a1c7f5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 23:14:27 +0200 Subject: [PATCH 6/6] fix(pipeline): preserve allocation policy in resolver cleanup Use tracked zeroed storage for the parallel LSP failure array and release it with the matching memory class after worker joins. Share import-filter failure cleanup without changing fallback state. Keep existing raw-allocation allowances unchanged. Signed-off-by: Martin Vogel --- src/pipeline/pass_lsp_cross.c | 9 +++++---- src/pipeline/pass_parallel.c | 9 +++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 643d3ff191..3ae60db0e5 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -2213,8 +2213,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); for (int i = 0; i < imp_count; i++) { if (!pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total)) { - free(selected); - return NULL; /* out_success remains false: dispatcher uses all_defs */ + goto failed; /* out_success remains false: dispatcher uses all_defs */ } } if (pxc_is_jvm_lang(caller_lang) && idx->namespace_ht) { @@ -2231,8 +2230,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef)); if (!out) { - free(selected); - return NULL; + goto failed; } int n = 0; @@ -2245,4 +2243,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef *out_success = true; free(selected); return out; +failed: + free(selected); + return NULL; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index af415e13b3..265017c507 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3877,6 +3877,10 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return 0; } + if (file_count < 0 || (size_t)file_count > SIZE_MAX / sizeof(const char *)) { + return CBM_NOT_FOUND; + } + cbm_log_info("parallel.resolve.start", "files", itoa_log(file_count), "workers", itoa_log(worker_count)); @@ -3886,7 +3890,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return CBM_NOT_FOUND; } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); - const char **lsp_failures = calloc((size_t)file_count, sizeof(*lsp_failures)); + const char **lsp_failures = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)file_count * sizeof(*lsp_failures)); if (!lsp_failures) { cbm_aligned_free(workers); return CBM_NOT_FOUND; @@ -3954,7 +3959,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], "lsp_skipped"); } - free(lsp_failures); + cbm_free(CBM_MEM_CLASS_OTHER, lsp_failures); /* Workers joined: the shared Rust registry (if built) is no longer read. * Free its dedicated arena + the lock (registry was self-contained: it strdup'd * all QNs, so freeing all_defs afterward is safe). */