From 519c765e1fd67ec12127ab41ec5f231401900ba8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 01:37:27 +0200 Subject: [PATCH 1/9] fix(python): retain typed fields across file boundaries Carry declared instance-field types through extraction and result compaction into the shared Python type registry so chained calls through imported classes resolve their receiver types. Refs #1277 Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 5 + internal/cbm/cbm.h | 19 ++++ internal/cbm/extract_defs.c | 167 ++++++++++++++++++++++++++++++++++ internal/cbm/lsp/py_lsp.c | 42 +++++++++ internal/cbm/result_compact.c | 7 ++ src/pipeline/pass_lsp_cross.c | 130 ++++++++++++++++++++++++++ src/pipeline/pipeline.c | 6 +- tests/test_extraction.c | 57 ++++++++++++ tests/test_pipeline.c | 93 +++++++++++++++++++ tests/test_py_lsp.c | 115 +++++++++++++++++++++++ 10 files changed, 639 insertions(+), 2 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 082bca1d6d..4d01d0c691 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -243,6 +243,11 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) arr->items[arr->count++] = ta; } +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = ft; +} + void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr) { GROW_ARRAY(arr, a); arr->items[arr->count++] = sr; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index fb5e446a5d..7563fa6b59 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -405,6 +405,17 @@ typedef struct { CBMChannelDirection direction; } CBMChannel; +/* Python: one annotated instance field of a class -- `x: T` or `x: T = v` in + * the class body, `self.x: T = v` in __init__, or `self.x = p` where `p` is an + * annotated __init__ parameter. Not a graph node: it only carries the field's + * declared type to the cross-file LSP, so `obj.x.m()` on a class imported + * from another file can be typed (#1277). */ +typedef struct { + const char *class_qn; // QN of the owning class + const char *field_name; // attribute name + const char *type_text; // raw annotation text, resolved later per file +} CBMFieldType; + // Rust: impl Trait for Struct typedef struct { const char *trait_name; // trait name (raw text) @@ -518,6 +529,12 @@ typedef struct { int cap; } CBMChannelArray; +typedef struct { + CBMFieldType *items; + int count; + int cap; +} CBMFieldTypeArray; + // Full extraction result for one file. typedef struct CBMFileResult { CBMArena arena; // owns local memory; composites may also retain child arenas below @@ -536,6 +553,7 @@ typedef struct CBMFileResult { CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation + CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) const char *module_qn; // module qualified name const char *namespace_name; // declared namespace/package (Java/Kotlin/C#/PHP), NULL if none @@ -892,6 +910,7 @@ void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw); void cbm_typerefs_push(CBMTypeRefArray *arr, CBMArena *a, CBMTypeRef tr); void cbm_envaccess_push(CBMEnvAccessArray *arr, CBMArena *a, CBMEnvAccess ea); void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta); +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index bb56e2870f..de5b2e90dd 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -4584,6 +4584,169 @@ static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const cha return true; } +/* ── Python annotated instance fields (#1277) ───────────────────── + * Exported as result->field_types (never graph nodes) so the cross-file LSP + * can type `obj.x` when obj's class lives in another file. Only DECLARED + * types count: a class-body annotation, a `self.x: T = v` annotation in + * __init__, or `self.x = p` where p is an annotated __init__ parameter. An + * unannotated right-hand side is never guessed at. */ + +static void py_push_field_type(CBMExtractCtx *ctx, const char *class_qn, TSNode name_node, + const char *type_text) { + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (!name || !name[0] || !type_text || !type_text[0]) { + return; + } + CBMFieldType ft = {.class_qn = class_qn, .field_name = name, .type_text = type_text}; + cbm_fieldtype_push(&ctx->result->field_types, ctx->arena, ft); +} + +/* The assignment inside an expression_statement, or a null node. */ +static TSNode py_statement_assignment(TSNode stmt) { + TSNode null_node = {0}; + if (strcmp(ts_node_type(stmt), "expression_statement") != 0 || + ts_node_named_child_count(stmt) == 0) { + return null_node; + } + TSNode inner = ts_node_named_child(stmt, 0); + return strcmp(ts_node_type(inner), "assignment") == 0 ? inner : null_node; +} + +/* Annotation text of the __init__ parameter called `name`, or NULL. */ +static const char *py_init_param_annotation(CBMExtractCtx *ctx, TSNode params, const char *name) { + uint32_t n = ts_node_named_child_count(params); + for (uint32_t i = 0; i < n; i++) { + TSNode p = ts_node_named_child(params, i); + const char *pk = ts_node_type(p); + TSNode pname = {0}; + if (strcmp(pk, "typed_default_parameter") == 0) { + pname = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(pk, "typed_parameter") == 0 && ts_node_named_child_count(p) > 0) { + pname = ts_node_named_child(p, 0); + } else { + continue; + } + TSNode ptype = ts_node_child_by_field_name(p, TS_FIELD("type")); + if (ts_node_is_null(pname) || ts_node_is_null(ptype) || + strcmp(ts_node_type(pname), "identifier") != 0) { + continue; + } + char *pn = cbm_node_text(ctx->arena, pname, ctx->source); + if (pn && strcmp(pn, name) == 0) { + return cbm_node_text(ctx->arena, ptype, ctx->source); + } + } + return NULL; +} + +/* `self.x: T = v` / `self.x = p` inside __init__ (self = its first parameter). */ +static void py_init_field_assignment(CBMExtractCtx *ctx, const char *class_qn, TSNode assign, + TSNode params, const char *self_name) { + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + if (ts_node_is_null(left) || strcmp(ts_node_type(left), "attribute") != 0) { + return; + } + TSNode obj = ts_node_child_by_field_name(left, TS_FIELD("object")); + TSNode attr = ts_node_child_by_field_name(left, TS_FIELD("attribute")); + if (ts_node_is_null(obj) || ts_node_is_null(attr) || + strcmp(ts_node_type(obj), "identifier") != 0) { + return; + } + char *obj_name = cbm_node_text(ctx->arena, obj, ctx->source); + if (!obj_name || strcmp(obj_name, self_name) != 0) { + return; + } + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(ann)) { + py_push_field_type(ctx, class_qn, attr, cbm_node_text(ctx->arena, ann, ctx->source)); + return; + } + TSNode right = ts_node_child_by_field_name(assign, TS_FIELD("right")); + if (ts_node_is_null(right) || strcmp(ts_node_type(right), "identifier") != 0) { + return; + } + char *rhs = cbm_node_text(ctx->arena, right, ctx->source); + if (rhs && strcmp(rhs, self_name) != 0) { + py_push_field_type(ctx, class_qn, attr, py_init_param_annotation(ctx, params, rhs)); + } +} + +/* Every `self.x` field assignment in one __init__ body, nested blocks + * included; nested functions, lambdas and classes have their own `self`. */ +static void py_extract_init_fields(CBMExtractCtx *ctx, const char *class_qn, TSNode fn) { + TSNode params = ts_node_child_by_field_name(fn, TS_FIELD("parameters")); + TSNode body = ts_node_child_by_field_name(fn, TS_FIELD("body")); + if (ts_node_is_null(params) || ts_node_is_null(body) || + ts_node_named_child_count(params) == 0) { + return; + } + TSNode self_node = ts_node_named_child(params, 0); + if (strcmp(ts_node_type(self_node), "identifier") != 0) { + return; + } + char *self_name = cbm_node_text(ctx->arena, self_node, ctx->source); + if (!self_name || !self_name[0]) { + return; + } + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_32); + ts_nstack_push(&stack, body); + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); + const char *k = ts_node_type(n); + if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || + strcmp(k, "lambda") == 0) { + continue; + } + TSNode assign = py_statement_assignment(n); + if (!ts_node_is_null(assign)) { + py_init_field_assignment(ctx, class_qn, assign, params, self_name); + continue; + } + /* Reverse push keeps source order on pop. */ + for (uint32_t i = ts_node_named_child_count(n); i > 0; i--) { + ts_nstack_push(&stack, ts_node_named_child(n, i - 1)); + } + } +} + +static void extract_py_field_types(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn) { + TSNode body = ts_node_child_by_field_name(class_node, TS_FIELD("body")); + if (ts_node_is_null(body)) { + return; + } + uint32_t count = ts_node_named_child_count(body); + /* Class-body annotations first: `x: T` / `x: T = v`. */ + for (uint32_t i = 0; i < count; i++) { + TSNode assign = py_statement_assignment(ts_node_named_child(body, i)); + if (ts_node_is_null(assign)) { + continue; + } + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(left) && !ts_node_is_null(ann) && + strcmp(ts_node_type(left), "identifier") == 0) { + py_push_field_type(ctx, class_qn, left, cbm_node_text(ctx->arena, ann, ctx->source)); + } + } + /* Then __init__. */ + for (uint32_t i = 0; i < count; i++) { + TSNode fn = ts_node_named_child(body, i); + if (strcmp(ts_node_type(fn), "decorated_definition") == 0) { + fn = ts_node_child_by_field_name(fn, TS_FIELD("definition")); + } + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "function_definition") != 0) { + continue; + } + TSNode fname = ts_node_child_by_field_name(fn, TS_FIELD("name")); + char *fn_name = + ts_node_is_null(fname) ? NULL : cbm_node_text(ctx->arena, fname, ctx->source); + if (fn_name && strcmp(fn_name, "__init__") == 0) { + py_extract_init_fields(ctx, class_qn, fn); + } + } +} + static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; const char *kind = ts_node_type(node); @@ -4970,6 +5133,10 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // Extract class-level variables (field declarations) extract_class_variables(ctx, node, class_qn, spec); + if (ctx->language == CBM_LANG_PYTHON) { + extract_py_field_types(ctx, node, class_qn); + } + // C# 12 primary-constructor parameters: declared on the class line // (`class Foo(IBar bar, IBaz baz) : Base { ... }`) and bound to implicit // captured fields accessible from any instance member. Tree-sitter c-sharp diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 2cf66ad012..70de5dce78 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -4965,6 +4965,47 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) { return out; } +/* Split a class def's field_defs ("name:QN|name:QN", built by the Python + * field fold in pass_lsp_cross.c) into the parallel NULL-terminated arrays a + * CBMRegisteredType carries. The types are already project QNs, so each one + * becomes a NAMED type verbatim. Without this, `obj.field.method()` on a class + * from another file lost the field's type (#1277); same-file classes get their + * fields from the per-file walk instead. Pure string work: no registry lookup + * happens here, so it is safe inside the type-registration pass. */ +static void py_split_field_defs(CBMArena *arena, const char *field_defs, const char ***names_out, + const CBMType ***types_out) { + *names_out = NULL; + *types_out = NULL; + const char **pairs = py_split_pipe(arena, field_defs); + if (!pairs) + return; + int n = 0; + while (pairs[n]) + n++; + const char **names = + (const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *)); + const CBMType **types = + (const CBMType **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const CBMType *)); + if (!names || !types) + return; + int kept = 0; + for (int i = 0; i < n; i++) { + char *colon = strchr(pairs[i], ':'); + if (!colon || colon == pairs[i] || !colon[1]) + continue; + *colon = '\0'; /* pairs[i] is this call's own arena copy */ + names[kept] = pairs[i]; + types[kept] = cbm_type_named(arena, colon + 1); + kept++; + } + if (kept == 0) + return; + names[kept] = NULL; + types[kept] = NULL; + *names_out = names; + *types_out = types; +} + /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, @@ -4990,6 +5031,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe if (d->method_names_str && d->method_names_str[0]) { rt.method_names = py_split_pipe(arena, d->method_names_str); } + py_split_field_defs(arena, d->field_defs, &rt.field_names, &rt.field_types); cbm_registry_add_type(reg, rt); } } diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 90c8c6d820..82f661d053 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -377,6 +377,12 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->channels.items[i].transport); cr_str(c, &r->channels.items[i].enclosing_func_qn); } + cr_array(c, (void **)&r->field_types.items, r->field_types.count, sizeof(CBMFieldType)); + for (int i = 0; i < r->field_types.count && r->field_types.items; i++) { + cr_str(c, &r->field_types.items[i].class_qn); + cr_str(c, &r->field_types.items[i].field_name); + cr_str(c, &r->field_types.items[i].type_text); + } cr_str(c, &r->module_qn); cr_str(c, &r->namespace_name); cr_list(c, &r->exports); @@ -523,6 +529,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.string_refs.cap = tmp.string_refs.count; tmp.infra_bindings.cap = tmp.infra_bindings.count; tmp.channels.cap = tmp.channels.count; + tmp.field_types.cap = tmp.field_types.count; /* A composite kept its per-unit results only so shallow-copied strings * stayed valid; every string is now a copy of its own. */ diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 3b74cc1ebb..66751a6829 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -529,6 +529,132 @@ static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *resu } } +/* Python annotation text -> the one type name it declares, or NULL. Strips a + * string-literal forward reference and unwraps `Optional[X]`, `X | None` and + * `None | X`; anything else with brackets, commas or a real union (generics, + * Callable, Union[A, B]) names no single receiver type and is dropped. */ +static const char *pxc_py_annotation_type_name(CBMArena *arena, const char *text) { + if (!text) { + return NULL; + } + while (*text == ' ') { + text++; + } + size_t n = strlen(text); + while (n > 0 && text[n - 1] == ' ') { + n--; + } + if (n >= 2 && (text[0] == '"' || text[0] == '\'') && text[n - 1] == text[0]) { + text++; + n -= 2; + } + static const char *const optional_prefixes[] = {"Optional[", "typing.Optional["}; + for (size_t i = 0; i < sizeof(optional_prefixes) / sizeof(optional_prefixes[0]); i++) { + size_t plen = strlen(optional_prefixes[i]); + if (n > plen + 1 && strncmp(text, optional_prefixes[i], plen) == 0 && text[n - 1] == ']') { + text += plen; + n -= plen + 1; + break; + } + } + static const char none_tail[] = " | None"; + static const char none_head[] = "None | "; + const size_t none_len = sizeof(none_tail) - 1; + if (n > none_len && strncmp(text + n - none_len, none_tail, none_len) == 0) { + n -= none_len; + } else if (n > none_len && strncmp(text, none_head, none_len) == 0) { + text += none_len; + n -= none_len; + } + if (n == 0) { + return NULL; + } + for (size_t i = 0; i < n; i++) { + char c = text[i]; + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '.'; + if (!ok) { + return NULL; + } + } + return cbm_arena_strndup(arena, text, n); +} + +/* Python: fold the file's annotated instance fields (result->field_types, + * #1277) into their class defs' field_defs as "name:QN|name:QN", the format + * py_register_lsp_defs reads. Each annotation resolves through the same + * registry + import map, and the same weak-strategy and type-like vetoes, as + * the file's base classes (pxc_resolve_base_qn): a field whose type is not a + * confidently-known project type is left out rather than guessed. Linear in + * the file: one hash of the file's class defs, one pass over its fields. */ +typedef struct { + int owner; /* index into defs[], or -1 */ + const char *type_qn; +} pxc_py_field_t; + +static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs, + int start, int end, const cbm_registry_t *reg, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + int nf = result ? result->field_types.count : 0; + if (!arena || nf == 0 || !defs || start >= end || !reg) { + return; + } + CBMHashTable *by_qn = cbm_ht_create((uint32_t)(end - start)); + pxc_py_field_t *fields = (pxc_py_field_t *)cbm_arena_alloc(arena, (size_t)nf * sizeof(*fields)); + /* Per class def: exact buffer size, then the write cursor into it. */ + size_t *lens = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + size_t *used = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + if (!by_qn || !fields || !lens || !used) { + cbm_ht_free(by_qn); /* NULL-safe */ + return; + } + memset(lens, 0, (size_t)(end - start) * sizeof(size_t)); + memset(used, 0, (size_t)(end - start) * sizeof(size_t)); + for (int si = start; si < end; si++) { + if (defs[si].label && strcmp(defs[si].label, "Class") == 0 && defs[si].qualified_name) { + cbm_ht_set(by_qn, defs[si].qualified_name, &defs[si]); + } + } + for (int f = 0; f < nf; f++) { + const CBMFieldType *ft = &result->field_types.items[f]; + fields[f].owner = -1; + fields[f].type_qn = NULL; + CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; + const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; + const char *qn = + name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + if (!qn || !ft->field_name || !ft->field_name[0]) { + continue; + } + fields[f].owner = (int)(owner - defs); + fields[f].type_qn = qn; + /* "name:QN" plus one byte for the '|' or the NUL after it. */ + lens[fields[f].owner - start] += strlen(ft->field_name) + 1 + strlen(qn) + 1; + } + cbm_ht_free(by_qn); + for (int si = start; si < end; si++) { + char *buf = lens[si - start] ? (char *)cbm_arena_alloc(arena, lens[si - start]) : NULL; + if (buf) { + buf[0] = '\0'; + } + defs[si].field_defs = buf; + } + for (int f = 0; f < nf; f++) { + char *buf = fields[f].owner >= 0 ? (char *)defs[fields[f].owner].field_defs : NULL; + if (!buf) { + continue; + } + int slot = fields[f].owner - start; + int wrote = + snprintf(buf + used[slot], lens[slot] - used[slot], "%s%s:%s", used[slot] ? "|" : "", + result->field_types.items[f].field_name, fields[f].type_qn); + if (wrote > 0) { + used[slot] += (size_t)wrote; + } + } +} + /* Carry one Rust type-level impl independently of any method definition. * `impl Trait for Type {}` is semantically meaningful even when the block is * empty (the trait may provide defaults), so attaching the relation only to @@ -634,6 +760,10 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are idx++; } } + if (files[fi].language == CBM_LANG_PYTHON) { + pxc_fold_py_field_types(arena, fr, defs, file_start, idx, base_reg, def_modules[fi], + imp_keys, imp_vals, imp_count); + } cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index ba41b877c8..32f9df6a52 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1156,7 +1156,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.count * sizeof(CBMStringRef) + (size_t)r->impl_traits.count * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) + - (size_t)r->channels.count * sizeof(CBMChannel); + (size_t)r->channels.count * sizeof(CBMChannel) + + (size_t)r->field_types.count * sizeof(CBMFieldType); cap_other += (size_t)r->imports.cap * sizeof(CBMImport) + (size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) + (size_t)r->throws.cap * sizeof(CBMThrow) + @@ -1165,7 +1166,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.cap * sizeof(CBMStringRef) + (size_t)r->impl_traits.cap * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) + - (size_t)r->channels.cap * sizeof(CBMChannel); + (size_t)r->channels.cap * sizeof(CBMChannel) + + (size_t)r->field_types.cap * sizeof(CBMFieldType); n_defs += (size_t)r->defs.count; n_calls += (size_t)r->calls.count; n_usages += (size_t)r->usages.count; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0b6800cd1f..4980426a92 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1182,6 +1182,62 @@ TEST(cpp_method_return_type_preserves_pointer_and_qualifier) { * ═══════════════════════════════════════════════════════════════════ */ /* --- Python --- */ +/* #1277: annotated Python instance fields reach result->field_types (the + * cross-file LSP's only source for them); nothing unannotated is guessed and + * no graph def is minted for a field. */ +static const char *py_field_type_of(const CBMFileResult *r, const char *cls_suffix, + const char *field) { + for (int i = 0; i < r->field_types.count; i++) { + const CBMFieldType *ft = &r->field_types.items[i]; + size_t cl = strlen(ft->class_qn); + size_t sl = strlen(cls_suffix); + if (cl >= sl && strcmp(ft->class_qn + cl - sl, cls_suffix) == 0 && + strcmp(ft->field_name, field) == 0) { + return ft->type_text; + } + } + return NULL; +} + +TEST(python_annotated_instance_fields_exported_issue1277) { + const char *src = "class Trainer(Base):\n" + " engine: Contract\n" + " limit: int = 3\n" + " plain = 5\n\n" + " def __init__(self, strategies: Contract, n, opt: 'Other' = None):\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.count = n\n" + " self.made = make()\n" + " self.opt = opt\n" + " if n:\n" + " self.nested: Nested = n\n" + " def inner(other: Inner):\n" + " self.hidden = other\n\n" + " def later(self, x: Later):\n" + " self.late = x\n"; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "trainer.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "engine"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "limit"), "int"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "strategies"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "typed"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "opt"), "'Other'"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "nested"), "Nested"); + /* Controls: no declared type, a nested function's binding, a non-__init__ method. */ + ASSERT_NULL(py_field_type_of(r, ".Trainer", "plain")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "count")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "made")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "hidden")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "late")); + ASSERT_EQ(r->field_types.count, 6); + for (int i = 0; i < r->defs.count; i++) { + ASSERT_FALSE(r->defs.items[i].label && strcmp(r->defs.items[i].label, "Field") == 0); + } + cbm_free_result(r); + PASS(); +} + TEST(python_function) { CBMFileResult *r = extract( "def greet(name):\n return f\"Hello {name}\"\n\ndef main():\n greet(\"World\")\n", @@ -8991,6 +9047,7 @@ SUITE(extraction) { /* Scripting */ RUN_TEST(python_function); RUN_TEST(python_class); + RUN_TEST(python_annotated_instance_fields_exported_issue1277); RUN_TEST(js_function); RUN_TEST(js_class); RUN_TEST(ts_function); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 56be6b88e3..b0ebc9f5a2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8166,6 +8166,98 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* #1277: typed instance fields of a Python class imported from another file. + * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an + * LSP strategy. */ +static int count_lsp_calls(cbm_store_t *s, const char *project, const char *caller_suffix, + const char *callee_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + size_t cs = strlen(caller_suffix); + size_t ts = strlen(callee_suffix); + if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0 && + edges[i].properties_json && strstr(edges[i].properties_json, "lsp")) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return hits; +} + +TEST(python_crossfile_typed_field_calls_issue1277) { + const char *contracts = "class Contract:\n" + " def process_batch(self) -> None:\n" + " ...\n"; + const char *trainer = "from contracts import Contract\n\n" + "def make():\n" + " return None\n\n" + "class Trainer:\n" + " engine: Contract\n\n" + " def __init__(self, strategies: Contract) -> None:\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.plain = make()\n"; + const char *loop = "from trainer import Trainer\n\n" + "def run(trainer: Trainer) -> None:\n" + " strategies = trainer.strategies\n" + " strategies.process_batch()\n\n" + "def run_typed(trainer: Trainer) -> None:\n" + " trainer.typed.process_batch()\n\n" + "def run_classlevel(trainer: Trainer) -> None:\n" + " trainer.engine.process_batch()\n\n" + "def run_untyped(trainer: Trainer) -> None:\n" + " trainer.plain.process_batch()\n"; + if (setup_usages_repo("contracts.py", contracts, "trainer.py", trainer) != 0) { + FAIL("failed to create temp dir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs(loop, f); + fclose(f); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); + cbm_pipeline_t *p = cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + + int via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + int via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + int via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + int via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_usages_repo(); + + ASSERT_EQ(via_alias, 1); /* self.x = */ + ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ + ASSERT_EQ(via_class, 1); /* class-level x: T */ + ASSERT_EQ(via_untyped, 0); /* control: no annotation, no guess */ + PASS(); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -16378,6 +16470,7 @@ SUITE(pipeline) { RUN_TEST(implements_creates_override); RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ + RUN_TEST(python_crossfile_typed_field_calls_issue1277); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee519..d65341900d 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -580,6 +580,119 @@ TEST(pylsp_crossfile_method_dispatch) { PASS(); } +/* Issue #1277 (reproducer contributed by @Enferlain): a method call through a + * typed instance field of a class imported from ANOTHER file. The field type + * reaches the resolver only through CBMLSPDef.field_defs on the class def. */ +TEST(pylsp_crossfile_receiver_through_typed_field) { + const char *source = "from trainer import Trainer\n" + "def run(trainer: Trainer):\n" + " strategies = trainer.strategies\n" + " return strategies.process_batch()\n"; + + CBMLSPDef defs[3]; + memset(defs, 0, sizeof(defs)); + + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract"; + + const char *imp_names[] = {"Trainer"}; + const char *imp_qns[] = {"trainer.Trainer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.loop", defs, 3, imp_names, + imp_qns, 1, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "run", "process_batch"), 0); + cbm_arena_destroy(&arena); + PASS(); +} + +/* #1277 on the production path: the sealed shared Tier-2 registry. Covers the + * direct chain, the local alias, a field inherited from a base class, and two + * controls: an undeclared field and a field whose type has no such method + * must stay unresolved (never guessed). */ +TEST(pylsp_crossfile_typed_field_shared_registry) { + CBMArena arena; + cbm_arena_init(&arena); + + CBMLSPDef defs[5]; + memset(defs, 0, sizeof(defs)); + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract|name:builtins.str"; + defs[3].qualified_name = "trainer.SubTrainer"; + defs[3].short_name = "SubTrainer"; + defs[3].label = "Class"; + defs[3].def_module_qn = "trainer"; + defs[3].embedded_types = "trainer.Trainer"; + defs[4].qualified_name = "trainer.Other"; + defs[4].short_name = "Other"; + defs[4].label = "Class"; + defs[4].def_module_qn = "trainer"; + for (int i = 0; i < 5; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 5); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + + const char *src = "from trainer import Trainer, SubTrainer, Other\n" + "def direct(t: Trainer):\n" + " return t.strategies.process_batch()\n" + "def alias(t: Trainer):\n" + " s = t.strategies\n" + " return s.process_batch()\n" + "def inherited(t: SubTrainer):\n" + " return t.strategies.process_batch()\n" + "def undeclared(t: Other):\n" + " return t.strategies.process_batch()\n" + "def wrong_type(t: Trainer):\n" + " return t.name.process_batch()\n"; + const char *imp_names[] = {"Trainer", "SubTrainer", "Other"}; + const char *imp_qns[] = {"trainer.Trainer", "trainer.SubTrainer", "trainer.Other"}; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test.loop", reg, imp_names, + imp_qns, 3, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "direct", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "alias", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "inherited", "Contract.process_batch"), 0); + ASSERT_EQ(find_resolved_arr(&out, "undeclared", "process_batch"), -1); + ASSERT_EQ(find_resolved_arr(&out, "wrong_type", "process_batch"), -1); + + cbm_arena_destroy(&arena); + PASS(); +} + /* Graph-quality guard for the Python seal fix (per-file field overlay). In the * FUSED path the shared Tier-2 registry is read_only, so `self.b = Bar()` can no * longer be recorded by mutating the shared registry; it is recorded in the per-file @@ -2193,6 +2306,8 @@ SUITE(py_lsp) { RUN_TEST(pylsp_pep695_generic_class); /* Phase 9 — cross-file + batch */ RUN_TEST(pylsp_crossfile_method_dispatch); + RUN_TEST(pylsp_crossfile_receiver_through_typed_field); + RUN_TEST(pylsp_crossfile_typed_field_shared_registry); RUN_TEST(pylsp_fused_self_attr_chain_via_overlay); RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228); RUN_TEST(pylsp_crossfile_inheritance); From bf2daa36036745b6b4bac5b118cd2ba778efa30c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 18:25:03 +0200 Subject: [PATCH 2/9] fix(lsp): detach field arrays before overlay refinement Copy inherited field-type pointer arrays into the writable registry arena before Python refines an existing field. Keep shared type objects intact and avoid retaining per-file scratch types in the sealed fallback. Add deterministic shared-state and later-overlay coverage, retaining the sequential Python field regression and adding a forced-parallel variant. Refs #1277 Signed-off-by: Martin Vogel --- internal/cbm/lsp/type_registry.c | 22 +++++- tests/test_c_lsp.c | 115 +++++++++++++++++++++++++++++++ tests/test_pipeline.c | 86 ++++++++++++++++++----- 3 files changed, 203 insertions(+), 20 deletions(-) diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 4e2f02012e..06b55c391d 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -677,8 +677,28 @@ CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const cha for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { const CBMRegisteredType *base = lookup_type_self(r, qualified_name); if (base) { + CBMRegisteredType copy = *base; + /* Field refinements overwrite entries in this array. The type + * objects remain immutable, but the writable overlay must own + * the pointer array rather than mutate its shared fallback. */ + if (base->field_types) { + size_t count = 0; + /* Types are parallel to names; a named field may have no + * type, so an interior NULL does not terminate that array. */ + while (base->field_names ? base->field_names[count] != NULL + : base->field_types[count] != NULL) { + count++; + } + const CBMType **field_types = (const CBMType **)cbm_arena_alloc( + head->arena, (count + 1) * sizeof(*field_types)); + if (!field_types) { + return NULL; + } + memcpy(field_types, base->field_types, (count + 1) * sizeof(*field_types)); + copy.field_types = field_types; + } int before = head->type_count; - cbm_registry_add_type(head, *base); + cbm_registry_add_type(head, copy); if (head->type_count == before + 1) { return &head->types[before]; } diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index 933cacc36f..d890911506 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -15350,6 +15350,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { memset(&t, 0, sizeof(t)); t.qualified_name = "pkg.T"; t.short_name = "T"; + const char *field_names[] = {"untyped", "typed", NULL}; + const CBMType *field_types[] = {NULL, cbm_type_named(&arena, "pkg.Value"), NULL}; + t.field_names = field_names; + t.field_types = field_types; cbm_registry_add_type(&base, t); cbm_registry_finalize(&base); base.read_only = true; @@ -15414,6 +15418,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T"); ASSERT_NOT_NULL(wt); ASSERT_EQ(overlay.type_count, 1); + /* A missing type for an earlier named field must not truncate the copy. */ + bool field_slots_preserved = wt->field_types && wt->field_types != field_types && + wt->field_types[0] == NULL && + wt->field_types[1] == field_types[1] && wt->field_types[2] == NULL; cbm_registry_all_types_chain(&overlay, &ti); seen = 0; while (cbm_type_short_iter_next(&ti) >= 0) { @@ -15430,6 +15438,7 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { cbm_arena_destroy(&scratch); cbm_arena_destroy(&arena); + ASSERT_TRUE(field_slots_preserved); PASS(); } @@ -15804,6 +15813,111 @@ TEST(seal_py_shared_registry_readonly_fields) { PASS(); } +/* #1277: the production resolver receives a writable overlay, so refining an + * existing field must detach its pointer array from the sealed fallback. */ +TEST(seal_py_overlay_existing_field_keeps_shared_type) { + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef defs[4] = {0}; + defs[0].qualified_name = "test.mod.Foo"; + defs[0].short_name = "Foo"; + defs[0].label = "Class"; + defs[0].def_module_qn = "test.mod"; + defs[0].field_defs = "x:test.contracts.Contract"; + defs[1].qualified_name = "test.mod.Foo.m"; + defs[1].short_name = "m"; + defs[1].label = "Method"; + defs[1].receiver_type = "test.mod.Foo"; + defs[1].def_module_qn = "test.mod"; + defs[2].qualified_name = "test.contracts.Contract"; + defs[2].short_name = "Contract"; + defs[2].label = "Class"; + defs[2].def_module_qn = "test.contracts"; + defs[3].qualified_name = "test.contracts.Contract.process_batch"; + defs[3].short_name = "process_batch"; + defs[3].label = "Method"; + defs[3].receiver_type = "test.contracts.Contract"; + defs[3].def_module_qn = "test.contracts"; + for (int i = 0; i < 4; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 4); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + const CBMRegisteredType *base = cbm_registry_lookup_type(reg, "test.mod.Foo"); + ASSERT_NOT_NULL(base); + ASSERT_NOT_NULL(base->field_names); + ASSERT_NOT_NULL(base->field_types); + const char **names_before = base->field_names; + const CBMType **types_before = base->field_types; + const CBMType *type_before = base->field_types[0]; + ASSERT_NOT_NULL(type_before); + ASSERT_NULL(base->field_types[1]); + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *writer = "class Foo:\n" + " def m(self):\n" + " self.x = 1\n"; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&scratch, writer, (int)strlen(writer), "test.mod", &overlay, + NULL, NULL, 0, NULL, &out, NULL); + + /* Observe while scratch is live, then release both arenas before any + * failure assertion so the regression also fails cleanly under LSan. */ + bool shared_names_unchanged = base->field_names == names_before; + bool shared_types_unchanged = base->field_types == types_before; + bool shared_element_unchanged = base->field_types[0] == type_before; + bool shared_terminator_unchanged = base->field_types[1] == NULL; + const CBMRegisteredType *refined = cbm_registry_lookup_type(&overlay, "test.mod.Foo"); + bool local_refined = refined && refined != base && refined->field_types && + refined->field_types != types_before && refined->field_types[0] && + refined->field_types[0] != type_before && + refined->field_types[0]->kind == CBM_TYPE_BUILTIN && + refined->field_types[1] == NULL; + cbm_arena_destroy(&scratch); + + /* Never read a corrupted shared pointer after scratch destruction. When + * intact, a later file must still see the original Contract field. */ + int hits = 0; + if (shared_names_unchanged && shared_types_unchanged && shared_element_unchanged && + shared_terminator_unchanged && local_refined) { + cbm_arena_init(&scratch); + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *reader = "from mod import Foo\n\n" + "def run(foo: Foo):\n" + " foo.x.process_batch()\n"; + const char *import_names[] = {"Foo"}; + const char *import_qns[] = {"test.mod.Foo"}; + memset(&out, 0, sizeof(out)); + cbm_run_py_lsp_cross_with_registry(&scratch, reader, (int)strlen(reader), "test.reader", + &overlay, import_names, import_qns, 1, NULL, &out, NULL); + for (int i = 0; i < out.count; i++) { + const CBMResolvedCall *call = &out.items[i]; + if (call->caller_qn && call->callee_qn && + strcmp(call->caller_qn, "test.reader.run") == 0 && + strcmp(call->callee_qn, "test.contracts.Contract.process_batch") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && + call->confidence >= CBM_LSP_CONFIDENCE_FLOOR) { + hits++; + } + } + cbm_arena_destroy(&scratch); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(shared_names_unchanged); + ASSERT_TRUE(shared_types_unchanged); + ASSERT_TRUE(shared_element_unchanged); + ASSERT_TRUE(shared_terminator_unchanged); + ASSERT_TRUE(local_refined); + ASSERT_EQ(hits, 1); + PASS(); +} + TEST(seal_cs_shared_registry_readonly) { CBMArena arena; cbm_arena_init(&arena); @@ -16402,6 +16516,7 @@ SUITE(c_lsp) { RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write); RUN_TEST(clsp_method_return_refinement_is_copy_on_write); RUN_TEST(seal_py_shared_registry_readonly_fields); + RUN_TEST(seal_py_overlay_existing_field_keeps_shared_type); RUN_TEST(seal_cs_shared_registry_readonly); RUN_TEST(seal_ts_shared_registry_readonly); RUN_TEST(seal_go_shared_registry_readonly); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index b0ebc9f5a2..9e26829e27 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8198,7 +8198,7 @@ static int count_lsp_calls(cbm_store_t *s, const char *project, const char *call return hits; } -TEST(python_crossfile_typed_field_calls_issue1277) { +static int check_python_crossfile_typed_field_calls_issue1277(bool parallel) { const char *contracts = "class Contract:\n" " def process_batch(self) -> None:\n" " ...\n"; @@ -8226,31 +8226,70 @@ TEST(python_crossfile_typed_field_calls_issue1277) { } char path[512]; snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); - FILE *f = fopen(path, "w"); - ASSERT_NOT_NULL(f); - fputs(loop, f); - fclose(f); + if (th_write_file(path, loop) != 0) { + teardown_usages_repo(); + FAIL("failed to write loop.py"); + } + if (parallel) { + /* Three semantic files plus 50 inert files exceed the production + * threshold for the fused parallel pipeline. */ + for (int i = 0; i < 50; i++) { + snprintf(path, sizeof(path), "%s/field_pad_%02d.py", g_usages_tmpdir, i); + if (th_write_file(path, "field_padding = 0\n") != 0) { + teardown_usages_repo(); + FAIL("failed to write parallel-selection fixture"); + } + } + } + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + FAIL("failed to save pipeline environment"); + } + int pin_workers_rc = parallel ? cbm_setenv("CBM_WORKERS", "4", 1) : 0; + int pin_single_rc = parallel ? cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") : 0; char db_path[512]; snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); - cbm_pipeline_t *p = cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL); - ASSERT_NOT_NULL(p); - ASSERT_EQ(cbm_pipeline_run(p), 0); - cbm_store_t *s = cbm_store_open_path(db_path); - ASSERT_NOT_NULL(s); - const char *project = cbm_pipeline_project_name(p); - - int via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); - int via_typed = - count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); - int via_class = - count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); - int via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_pipeline_t *p = pin_workers_rc == 0 && pin_single_rc == 0 + ? cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL) + : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; + bool store_opened = s != NULL; + int via_alias = -1, via_typed = -1, via_class = -1, via_untyped = -1; + if (s) { + const char *project = cbm_pipeline_project_name(p); + via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_store_close(s); + } - cbm_store_close(s); cbm_pipeline_free(p); + int restore_workers_rc = + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + int restore_single_rc = saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); teardown_usages_repo(); + ASSERT_EQ(pin_workers_rc, 0); + ASSERT_EQ(pin_single_rc, 0); + ASSERT_EQ(restore_workers_rc, 0); + ASSERT_EQ(restore_single_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_TRUE(store_opened); ASSERT_EQ(via_alias, 1); /* self.x = */ ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ ASSERT_EQ(via_class, 1); /* class-level x: T */ @@ -8258,6 +8297,14 @@ TEST(python_crossfile_typed_field_calls_issue1277) { PASS(); } +TEST(python_crossfile_typed_field_calls_issue1277) { + return check_python_crossfile_typed_field_calls_issue1277(false); +} + +TEST(python_crossfile_typed_field_calls_issue1277_parallel) { + return check_python_crossfile_typed_field_calls_issue1277(true); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -16471,6 +16518,7 @@ SUITE(pipeline) { RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ RUN_TEST(python_crossfile_typed_field_calls_issue1277); + RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); From e3535dc8a80a50120a2be48e07b05dd3e2dec544 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 20:11:45 +0200 Subject: [PATCH 3/9] fix(lsp): bound repeated resolver work without truncating large files Memoize C expression evaluation, index repeated scope and registry queries, and report incomplete walks explicitly on memo or depth failure. Refs #1527 Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 203 +++++++++++-- internal/cbm/cbm.h | 15 + internal/cbm/lsp/c_lsp.c | 404 +++++++++++++++++++++++--- internal/cbm/lsp/c_lsp.h | 58 +++- internal/cbm/lsp/kotlin_lsp.c | 9 +- internal/cbm/lsp/lsp_node_iter.h | 29 ++ internal/cbm/lsp/lsp_work.h | 24 ++ internal/cbm/lsp/py_lsp.c | 377 ++++++++++++++++++------ internal/cbm/lsp/py_lsp.h | 41 ++- internal/cbm/lsp/scope.c | 164 +++++++++-- internal/cbm/lsp/scope.h | 35 ++- internal/cbm/lsp/ts_lsp.c | 8 +- internal/cbm/lsp/type_registry.c | 21 +- internal/cbm/lsp/type_registry.h | 7 + src/pipeline/pass_lsp_cross.c | 46 ++- src/pipeline/pass_parallel.c | 38 ++- tests/test_c_lsp.c | 477 +++++++++++++++++++++++++++++++ tests/test_extraction.c | 46 +++ tests/test_pipeline.c | 182 ++++++++++++ tests/test_py_lsp.c | 463 +++++++++++++++++++++++++++++- tests/test_py_lsp_scale.c | 28 +- 21 files changed, 2416 insertions(+), 259 deletions(-) create mode 100644 internal/cbm/lsp/lsp_work.h diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 4d01d0c691..463c44074d 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -20,6 +20,7 @@ #include "lsp/java_lsp.h" #include "lsp/kotlin_lsp.h" #include "lsp/rust_lsp.h" +#include "lsp/lsp_work.h" #include "preprocessor.h" #include "sql_values.h" // #1735: literal INSERT rows kept out of the SQL parse #include "foundation/compat.h" @@ -2146,6 +2147,183 @@ static bool cbm_sql_values_exclusion_on(const char *rel_path) { return true; } +/* ── Innermost enclosing Function/Method per call (#1527) ────────────── + * The call-context metrics attribute each call to the Function/Method def with + * the SMALLEST line span containing the call's line, ties to the lowest def + * index. Scanning every def for every call made that O(calls x defs) per file + * -- quadratic in file size, the largest single cost on a module with + * thousands of methods. The sweep below gives the identical answer in + * O((calls + defs) log defs): calls in line order, candidates entering a + * min-heap keyed (span, index) once their start line is reached and leaving it + * lazily once their end line has passed -- lines only grow, so a def that has + * ended never contains a later call. The heap top is then the smallest + * (span, index) among the defs that contain the line. */ +static bool enclosing_is_callable(const CBMDefinition *d) { + return d->name && d->label && + (strcmp(d->label, "Function") == 0 || strcmp(d->label, "Method") == 0); +} + +/* The reference answer for one line (and the path when the sweep's arrays + * cannot be allocated). */ +static int cbm_enclosing_callable_scan(const CBMFileResult *result, int line) { + int best = -1; + int best_span = -1; + for (int di = 0; di < result->defs.count; di++) { + const CBMDefinition *d = &result->defs.items[di]; + CBM_LSP_WORK(1); + if (!enclosing_is_callable(d)) { + continue; + } + if ((int)d->start_line <= line && line <= (int)d->end_line) { + int span = (int)d->end_line - (int)d->start_line; + if (best < 0 || span < best_span) { + best_span = span; + best = di; + } + } + } + return best; +} + +static CBM_TLS const CBMFileResult *tl_enclosing_result; + +static int enclosing_def_by_start(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + uint32_t la = tl_enclosing_result->defs.items[ia].start_line; + uint32_t lb = tl_enclosing_result->defs.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +static int enclosing_call_by_line(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + int la = tl_enclosing_result->calls.items[ia].start_line; + int lb = tl_enclosing_result->calls.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +/* Heap order: smaller span first, then smaller def index. */ +static bool enclosing_heap_less(const CBMFileResult *r, int a, int b) { + const CBMDefinition *da = &r->defs.items[a]; + const CBMDefinition *db = &r->defs.items[b]; + int sa = (int)da->end_line - (int)da->start_line; + int sb = (int)db->end_line - (int)db->start_line; + return sa != sb ? sa < sb : a < b; +} + +static void enclosing_heap_push(const CBMFileResult *r, int *heap, int *n, int di) { + int i = (*n)++; + heap[i] = di; + while (i > 0) { + int parent = (i - 1) / 2; + CBM_LSP_WORK(1); + if (!enclosing_heap_less(r, heap[i], heap[parent])) { + break; + } + int t = heap[i]; + heap[i] = heap[parent]; + heap[parent] = t; + i = parent; + } +} + +static void enclosing_heap_pop(const CBMFileResult *r, int *heap, int *n) { + heap[0] = heap[--(*n)]; + int i = 0; + for (;;) { + int l = (2 * i) + 1; + int m = i; + CBM_LSP_WORK(1); + if (l < *n && enclosing_heap_less(r, heap[l], heap[m])) { + m = l; + } + if (l + 1 < *n && enclosing_heap_less(r, heap[l + 1], heap[m])) { + m = l + 1; + } + if (m == i) { + break; + } + int t = heap[i]; + heap[i] = heap[m]; + heap[m] = t; + i = m; + } +} + +/* enclosing[ci] = innermost Function/Method def index for call ci, or -1. + * Returns NULL when the working arrays cannot be allocated; the caller then + * asks cbm_enclosing_callable_scan per call (same answer, old cost). */ +static int *cbm_enclosing_callables(const CBMFileResult *result, int call_count) { + int def_count = result->defs.count; + if (call_count <= 0) { + return NULL; + } + size_t call_bytes = (size_t)call_count * sizeof(int); + size_t def_bytes = (size_t)(def_count > 0 ? def_count : 1) * sizeof(int); + int *enclosing = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *call_order = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *cands = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + int *heap = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + if (!enclosing || !call_order || !cands || !heap) { + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return NULL; + } + int ncand = 0; + for (int di = 0; di < def_count; di++) { + if (enclosing_is_callable(&result->defs.items[di])) { + cands[ncand++] = di; + } + } + for (int ci = 0; ci < call_count; ci++) { + call_order[ci] = ci; + enclosing[ci] = -1; + } + tl_enclosing_result = result; + qsort(cands, (size_t)ncand, sizeof(int), enclosing_def_by_start); + qsort(call_order, (size_t)call_count, sizeof(int), enclosing_call_by_line); + tl_enclosing_result = NULL; + + int heap_n = 0; + int next = 0; + for (int k = 0; k < call_count; k++) { + int ci = call_order[k]; + int line = result->calls.items[ci].start_line; + while (next < ncand && (int)result->defs.items[cands[next]].start_line <= line) { + enclosing_heap_push(result, heap, &heap_n, cands[next++]); + } + while (heap_n > 0 && (int)result->defs.items[heap[0]].end_line < line) { + enclosing_heap_pop(result, heap, &heap_n); + } + enclosing[ci] = heap_n > 0 ? heap[0] : -1; + } + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return enclosing; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result) { + int *enclosing = cbm_enclosing_callables(result, result->calls.count); + if (!enclosing) { + return result->calls.count > 0 ? -1 : 0; + } + int mismatches = 0; + for (int ci = 0; ci < result->calls.count; ci++) { + int line = result->calls.items[ci].start_line; + if (line > 0 && enclosing[ci] != cbm_enclosing_callable_scan(result, line)) { + mismatches++; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + return mismatches; +} +#endif + static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, @@ -2509,8 +2687,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Also run LSP on expanded source for additional type-resolved // calls (language is already C/C++/CUDA — checked in enclosing // block). Runs in every mode. - cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, - language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + if (!result->lsp_skipped) { + cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, + language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + } /* All C-LSP emitters stamp origin directly so rewrite-time * comparisons are already safe. Keep this boundary sweep as @@ -2833,28 +3013,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C d->param_count = pc; } + int *enclosing = cbm_enclosing_callables(result, orig_calls_count); for (int ci = 0; ci < orig_calls_count; ci++) { const CBMCall *c = &result->calls.items[ci]; if (!c->callee_name || c->start_line <= 0) { continue; } // Innermost enclosing Function/Method def by line range (smallest span). - int best = -1; - int best_span = -1; - for (int di = 0; di < def_count; di++) { - const CBMDefinition *d = &result->defs.items[di]; - if (!d->name || !d->label || - (strcmp(d->label, "Function") != 0 && strcmp(d->label, "Method") != 0)) { - continue; - } - if ((int)d->start_line <= c->start_line && c->start_line <= (int)d->end_line) { - int span = (int)d->end_line - (int)d->start_line; - if (best < 0 || span < best_span) { - best_span = span; - best = di; - } - } - } + int best = enclosing ? enclosing[ci] : cbm_enclosing_callable_scan(result, c->start_line); if (best < 0) { continue; } @@ -2899,6 +3065,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } free(has_self); free(has_guarded); + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); uint64_t t2 = now_ns(); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 7563fa6b59..6cf6d04fec 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -450,6 +450,14 @@ typedef struct { int cap; } CBMResolvedCallArray; +/* Refinement status: existing parser/no-op exits are separate from these + * explicit evaluator failures. The caller retains prior completed output. */ +typedef enum { + CBM_LSP_COMPLETE = 0, + CBM_LSP_MEMO_FAILED, + CBM_LSP_DEPTH_EXCEEDED, +} CBMLSPStatus; + // Growable arrays used during extraction. typedef struct { CBMDefinition *items; @@ -914,6 +922,13 @@ void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); + +#ifdef CBM_ENABLE_TEST_SEAMS +/* #1527 seam: how many of result's calls get a different innermost enclosing + * Function/Method from the call-context sweep than from the reference + * per-call scan (0 = identical), or -1 when the sweep could not allocate. */ +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result); +#endif void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 74b97d69d9..3506707615 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -1,10 +1,12 @@ #include "c_lsp.h" #include "lsp_node_iter.h" +#include "lsp_work.h" #include "../helpers.h" #include #include #include #include +#include /* Safe kind accessor — returns CBM_TYPE_UNKNOWN for NULL types. * Prevents SEGV in c_eval_expr_type_inner on unusual C++ AST shapes. */ @@ -12,6 +14,47 @@ static inline CBMTypeKind safe_kind(const CBMType *t) { return t ? t->kind : CBM_TYPE_UNKNOWN; } +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int c_test_memo_allocations_left = -1; +static _Thread_local bool c_test_depth_failure; +void cbm_c_lsp_test_depth_fail(bool enabled) { + c_test_depth_failure = enabled; +} +static _Thread_local uint32_t c_test_walks; +uint32_t cbm_c_lsp_test_walks_take(void) { + uint32_t walks = c_test_walks; + c_test_walks = 0; + return walks; +} +void cbm_c_lsp_test_memo_fail_after(int successful_allocations) { + c_test_memo_allocations_left = successful_allocations; +} +#endif + +static void c_memo_test_stage(CLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + c_test_walks++; + const char *depth_stage = getenv("CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 257; + const char *fail_stage = getenv("CBM_TEST_C_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime keeps the reason available even when the arena is full. + * Result compaction/spilling copies strings, including non-arena strings. */ + result->error_msg = + status == CBM_LSP_DEPTH_EXCEEDED ? CBM_C_LSP_DEPTH_ERROR : CBM_C_LSP_MEMO_ERROR; +} + // Forward declarations static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node); static void c_emit_resolved_call(CLSPContext *ctx, const char *callee_qn, const char *strategy, @@ -59,6 +102,11 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc const CBMTypeRegistry *registry, const char *module_qn, bool cpp_mode, CBMResolvedCallArray *out) { memset(ctx, 0, sizeof(CLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = c_test_memo_allocations_left; + if (c_test_depth_failure) + ctx->eval_depth = 257; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; @@ -191,14 +239,7 @@ static bool c_scope_declares_name(const CBMScope *scope, const char *var_name) { if (!scope || !var_name) { return false; } - for (const CBMScopeChunk *chunk = scope->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, var_name) == 0) { - return true; - } - } - } - return false; + return cbm_scope_lookup_local(scope, var_name) != NULL; } /* Return the exact fp-state entry for the nearest lexical declaration. An @@ -1474,26 +1515,129 @@ static const char *type_to_qn(const CBMType *t) { static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node); +/* Stack guard: the evaluator recurses once per expression nesting level. + * There is deliberately NO per-file work budget (#1527): the 10000-step cap + * that used to sit here (#323) left every later expression in a large file + * unresolved while the file still reported status `indexed`. What it was + * containing -- a fluent chain re-evaluating its receiver at every link, 2^N + * evaluations (the Bitcoin versionbits hang) -- is removed at the root by the + * per-evaluation memo below. */ #define C_EVAL_DEPTH_LIMIT 256 -#define C_EVAL_MAX_STEPS_PER_FILE 10000 +enum { C_EVAL_MEMO_INITIAL_CAP = 64 }; + +static uint32_t c_eval_memo_hash(const void *id) { + uintptr_t v = (uintptr_t)id >> 3; /* node structs are >=8-byte aligned */ + return (uint32_t)((v ^ (v >> 29)) * 2654435761u); +} + +/* Empty the memo for a new outermost evaluation: O(slots the last one used). */ +static void c_eval_memo_reset(CLSPContext *ctx) { + for (int i = 0; i < ctx->eval_memo_count; i++) { + ctx->eval_memo[ctx->eval_memo_used[i]].node_id = NULL; + } + ctx->eval_memo_count = 0; +} + +static const CBMType *c_eval_memo_lookup(const CLSPContext *ctx, const void *id) { + if (ctx->eval_memo_cap == 0) { + return NULL; + } + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + for (uint32_t i = c_eval_memo_hash(id) & mask;; i = (i + 1) & mask) { + const struct CLSPEvalMemoSlot *e = &ctx->eval_memo[i]; + if (!e->node_id) { + return NULL; + } + if (e->node_id == id) { + return e->result; + } + } +} + +static void c_eval_memo_place(CLSPContext *ctx, const void *id, const CBMType *result) { + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + uint32_t i = c_eval_memo_hash(id) & mask; + while (ctx->eval_memo[i].node_id) { + i = (i + 1) & mask; + } + ctx->eval_memo[i].node_id = id; + ctx->eval_memo[i].result = result; + ctx->eval_memo_used[ctx->eval_memo_count++] = (int)i; +} + +static void *c_eval_memo_alloc(CLSPContext *ctx, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return NULL; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif + return cbm_arena_alloc(ctx->arena, size); +} + +/* Memoization is required to bound receiver re-evaluation. An allocation + * failure stops this file's refinement and is reported to the caller. */ +static bool c_eval_memo_insert(CLSPContext *ctx, const void *id, const CBMType *result) { + if (ctx->eval_memo_count >= ctx->eval_memo_cap / 2) { + if (ctx->eval_memo_cap > INT_MAX / 2) + return false; + int new_cap = ctx->eval_memo_cap ? ctx->eval_memo_cap * 2 : C_EVAL_MEMO_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(struct CLSPEvalMemoSlot) || + (size_t)(new_cap / 2) > SIZE_MAX / sizeof(int)) + return false; + struct CLSPEvalMemoSlot *slots = c_eval_memo_alloc(ctx, (size_t)new_cap * sizeof(*slots)); + if (!slots) + return false; + int *used = c_eval_memo_alloc(ctx, (size_t)(new_cap / 2) * sizeof(int)); + if (!used) + return false; + memset(slots, 0, (size_t)new_cap * sizeof(*slots)); + struct CLSPEvalMemoSlot *old = ctx->eval_memo; + const int *old_used = ctx->eval_memo_used; + int old_count = ctx->eval_memo_count; + ctx->eval_memo = slots; + ctx->eval_memo_used = used; + ctx->eval_memo_cap = new_cap; + ctx->eval_memo_count = 0; + for (int k = 0; k < old_count; k++) { + const struct CLSPEvalMemoSlot *e = &old[old_used[k]]; + c_eval_memo_place(ctx, e->node_id, e->result); + } + } + c_eval_memo_place(ctx, id, result); + return true; +} const CBMType *c_eval_expr_type(CLSPContext *ctx, TSNode node) { - if (ts_node_is_null(node)) + if (ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); - /* Expression type evaluation is best-effort. Some recovery-mode C++ ASTs - * can repeatedly drive member/type lookup without increasing recursion - * depth. Keep a generous per-file work budget so pathological expressions - * degrade to unknown instead of hanging repository indexing. */ - if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT || ctx->eval_steps++ > C_EVAL_MAX_STEPS_PER_FILE) { - if (ctx->debug && ctx->eval_steps == C_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [clsp] expression eval step budget exhausted; returning unknown\n"); - } + if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT) { + ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } + if (ctx->eval_depth == 0) { + c_eval_memo_reset(ctx); /* scope may have changed since the last evaluation */ + } + const CBMType *cached = c_eval_memo_lookup(ctx, node.id); + if (cached) { + return cached; + } + CBM_LSP_WORK(1); /* one real evaluation (complexity tests) */ + uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; const CBMType *result = c_eval_expr_type_inner(ctx, node); ctx->eval_depth--; - return result ? result : cbm_type_unknown(); + if (!result) { + result = cbm_type_unknown(); + } + /* Never memoize a result a depth cutoff may have truncated: the incomplete walk + * must stop before a second receiver branch repeats the truncated work. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !c_eval_memo_insert(ctx, node.id, result)) { + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + } + return ctx->eval_failure ? cbm_type_unknown() : result; } static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node) { @@ -2963,6 +3107,8 @@ static const CBMType *c_parse_declaration_type(CLSPContext *ctx, TSNode decl_nod } void c_process_statement(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ts_node_is_null(node)) return; const char *kind = ts_node_type(node); @@ -3930,6 +4076,8 @@ static void c_resolve_calls_in_node_inner(CLSPContext *ctx, TSNode node); * frames via c_adl_resolve). Past the cap the subtree is skipped — its calls * stay unresolved, which is graceful degradation, not a crash. */ static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ctx->walk_depth >= C_LSP_MAX_WALK_DEPTH) return; ctx->walk_depth++; @@ -4783,6 +4931,8 @@ recurse:; // ============================================================================ static void c_process_function(CLSPContext *ctx, TSNode func_node) { + if (ctx->eval_failure) + return; TSNode decl = ts_node_child_by_field_name(func_node, "declarator", 10); if (ts_node_is_null(decl)) return; @@ -4973,6 +5123,8 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // Process a top-level or nested declaration within a namespace/class body, // handling template_declaration wrapping. static void c_process_body_child(CLSPContext *ctx, TSNode child) { + if (ctx->eval_failure) + return; if (ts_node_is_null(child)) return; const char *ck = ts_node_type(child); @@ -5429,6 +5581,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { if (ts_node_is_null(root)) return; + int resolved_before = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + // Collect top-level children once (O(n)); both passes reuse the array. // Indexing ts_node_child(root,i) per iteration is O(i) → O(n²) on a wide root. uint32_t kn = 0; @@ -5437,7 +5591,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { TSNode inner; // Pass 1: process using declarations and global variables - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; const char *ck = ts_node_type(child); @@ -5461,7 +5615,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { } // Pass 2: process functions, namespaces, classes, templates - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; c_process_body_child(ctx, child); } @@ -5469,6 +5623,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { // Release the per-file negative-lookup memo (malloc-owned; only allocated in // shared-registry mode). No-op when it was never populated. c_neg_memo_free(ctx); + if (ctx->eval_failure && ctx->resolved_calls) + ctx->resolved_calls->count = resolved_before; } // ============================================================================ @@ -5682,8 +5838,116 @@ static void c_rewrite_proved_destructor_candidates(CBMFileResult *result) { } } +/* QN -> index of its FIRST registration in reg->types, for the single-file + * registration loop below (#1527). That loop asked "is this type registered + * yet, and where?" by scanning every registered type per Field and per + * Method -- O(defs x types) before the registry was indexed. Tracking the + * first index as types are added answers the same question in O(1); the + * types array may be reallocated as it grows, so indices, not pointers. */ +typedef struct { + const char **qn; /* NULL = empty slot */ + int *first; + int cap; /* power of two, or 0 when unavailable */ + int count; +} CTypeFirstIndex; + +static uint32_t c_type_first_hash(const char *s) { + uint32_t h = 2166136261u; + for (const unsigned char *p = (const unsigned char *)s; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Slot holding qn, or the empty slot where it belongs. */ +static int c_type_first_slot(const CTypeFirstIndex *ix, const char *qn) { + uint32_t mask = (uint32_t)ix->cap - 1; + uint32_t i = c_type_first_hash(qn) & mask; + while (ix->qn[i] && strcmp(ix->qn[i], qn) != 0) { + CBM_LSP_WORK(1); + i = (i + 1) & mask; + } + return (int)i; +} + +static bool c_type_first_alloc(CTypeFirstIndex *ix, CBMArena *arena, int cap) { + const char **qn = (const char **)cbm_arena_alloc(arena, (size_t)cap * sizeof(*qn)); + int *first = (int *)cbm_arena_alloc(arena, (size_t)cap * sizeof(*first)); + if (!qn || !first) { + return false; + } + memset(qn, 0, (size_t)cap * sizeof(*qn)); + ix->qn = qn; + ix->first = first; + ix->cap = cap; + ix->count = 0; + return true; +} + +/* Record reg->types[idx]; an existing QN keeps its first index. On OOM the + * index turns itself off (cap 0) and callers fall back to the scan. */ +static void c_type_first_note(CTypeFirstIndex *ix, CBMArena *arena, const CBMTypeRegistry *reg, + int idx) { + const char *qn = reg->types[idx].qualified_name; + if (ix->cap == 0 || !qn) { + return; + } + if ((ix->count + 1) * 2 > ix->cap) { + CTypeFirstIndex grown = {0}; + if (!c_type_first_alloc(&grown, arena, ix->cap * 2)) { + ix->cap = 0; + return; + } + for (int i = 0; i < ix->cap; i++) { + if (ix->qn[i]) { + int s = c_type_first_slot(&grown, ix->qn[i]); + grown.qn[s] = ix->qn[i]; + grown.first[s] = ix->first[i]; + grown.count++; + } + } + *ix = grown; + } + int s = c_type_first_slot(ix, qn); + if (!ix->qn[s]) { + ix->qn[s] = qn; + ix->first[s] = idx; + ix->count++; + } +} + +/* Index of the first registered type named qn, or -1. Same answer as the + * linear scan it replaces (which also serves as the OOM fallback). */ +static int c_type_first_find(const CTypeFirstIndex *ix, const CBMTypeRegistry *reg, + const char *qn) { + if (ix->cap > 0) { + int s = c_type_first_slot(ix, qn); + return ix->qn[s] ? ix->first[s] : -1; + } + for (int ri = 0; ri < reg->type_count; ri++) { + CBM_LSP_WORK(1); + if (strcmp(reg->types[ri].qualified_name, qn) == 0) { + return ri; + } + } + return -1; +} + +/* cbm_registry_add_type plus the first-index bookkeeping. */ +static void c_add_type_indexed(CBMTypeRegistry *reg, CTypeFirstIndex *ix, CBMArena *arena, + CBMRegisteredType rt) { + int before = reg->type_count; + cbm_registry_add_type(reg, rt); + if (reg->type_count > before) { + c_type_first_note(ix, arena, reg, before); + } +} + void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin) { + if (result->lsp_skipped) + return; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -5695,6 +5959,18 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i const char *module_qn = result->module_qn; + /* Scratch for the registration index and the registry's hash index; both + * die with this call (quadratic-pattern #6: never per-file index memory in + * a longer-lived arena). */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + CTypeFirstIndex first_ix = {0}; + if (c_type_first_alloc(&first_ix, &idx_arena, 256)) { + for (int ti = 0; ti < reg.type_count; ti++) { + c_type_first_note(&first_ix, &idx_arena, ®, ti); + } + } + // Register file's own definitions for (int i = 0; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; @@ -5728,19 +6004,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i rt.embedded_types = embedded; } } - cbm_registry_add_type(®, rt); + c_add_type_indexed(®, &first_ix, &idx_arena, rt); } // Field definitions → populate field_names/field_types on parent type if (d->label && strcmp(d->label, "Field") == 0 && d->parent_class && d->return_type) { // Find or add the parent type in registry - CBMRegisteredType *parent_rt = NULL; - for (int ri = 0; ri < reg.type_count; ri++) { - if (strcmp(reg.types[ri].qualified_name, d->parent_class) == 0) { - parent_rt = ®.types[ri]; - break; - } - } + int parent_idx = c_type_first_find(&first_ix, ®, d->parent_class); + CBMRegisteredType *parent_rt = parent_idx >= 0 ? ®.types[parent_idx] : NULL; if (parent_rt) { // Count existing fields int existing = 0; @@ -5807,14 +6078,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i if (strcmp(d->label, "Method") == 0 && d->parent_class) { rf.receiver_type = d->parent_class; // Auto-create type if needed - if (!cbm_registry_lookup_type(®, d->parent_class)) { + if (c_type_first_find(&first_ix, ®, d->parent_class) < 0) { CBMRegisteredType auto_type; memset(&auto_type, 0, sizeof(auto_type)); auto_type.qualified_name = d->parent_class; // Extract short name const char *dot = strrchr(d->parent_class, '.'); auto_type.short_name = dot ? dot + 1 : d->parent_class; - cbm_registry_add_type(®, auto_type); + c_add_type_indexed(®, &first_ix, &idx_arena, auto_type); } } @@ -5822,13 +6093,27 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i } } + /* Hash-index the registry before the walk (#1527): unindexed, every + * resolver lookup scanned the stdlib plus all of this file's definitions, + * O(calls x defs) per file. First-registered chains keep each duplicated + * name resolving to the entry the scan returned; resolver additions after + * this point stay visible through the registry's tail scan. */ + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + // Initialize context and run CLSPContext ctx; c_lsp_init(&ctx, arena, source, source_len, ®, module_qn, cpp_mode, &result->resolved_calls); ctx.source_origin = source_origin; + c_memo_test_stage(&ctx, + source_origin == CBM_SOURCE_ORIGIN_PREPROCESSED ? "preprocessed" : "raw"); c_lsp_process_file(&ctx, root); - c_rewrite_proved_destructor_candidates(result); + if (ctx.eval_failure) + cbm_c_lsp_record_failure(result, ctx.eval_failure); + else + c_rewrite_proved_destructor_candidates(result); + cbm_arena_destroy(&idx_arena); } // ============================================================================ @@ -6000,13 +6285,22 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in return reg; } -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, cpp_mode, reg, include_paths, include_ns_qns, + include_count, cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, + int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -6014,14 +6308,14 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6031,20 +6325,32 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int for (int i = 0; i < include_count; i++) { c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_status(arena, source, source_len, module_qn, cpp_mode, defs, + def_count, include_paths, include_ns_qns, include_count, + cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out) - return; + return CBM_LSP_COMPLETE; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -6064,14 +6370,14 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6090,20 +6396,23 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } // --- Batch cross-file LSP --- -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchCLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -6119,10 +6428,12 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co memset(&file_out, 0, sizeof(file_out)); // Delegate to existing per-file function - cbm_run_c_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->cpp_mode, file->defs, file->def_count, file->include_paths, - file->include_ns_qns, file->include_count, file->cached_tree, - &file_out); + bool file_complete = cbm_run_c_lsp_cross( + &file_arena, file->source, file->source_len, file->module_qn, file->cpp_mode, + file->defs, file->def_count, file->include_paths, file->include_ns_qns, + file->include_count, file->cached_tree, &file_out); + + complete = complete && file_complete; // Copy results to output arena (must outlive per-file arena) if (file_out.count > 0) { @@ -6147,4 +6458,5 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co cbm_arena_destroy(&file_arena); } + return complete; } diff --git a/internal/cbm/lsp/c_lsp.h b/internal/cbm/lsp/c_lsp.h index af81d5bbdf..bbbf236555 100644 --- a/internal/cbm/lsp/c_lsp.h +++ b/internal/cbm/lsp/c_lsp.h @@ -99,12 +99,33 @@ typedef struct { // READ-ONLY across resolve workers — never mutate it // (and never store per-worker arena pointers into it) bool debug; - int eval_depth; // recursion depth for c_eval_expr_type (crash guard) - int eval_steps; // total expression eval calls for current file (hang guard) + int eval_depth; // recursion depth for c_eval_expr_type (crash guard) + uint32_t eval_truncations; // depth-cap cutoffs; a truncated result is never memoized + // Per-evaluation memo (#1527): results of the sub-expressions evaluated + // during ONE outermost c_eval_expr_type call, keyed by node identity. The + // evaluator re-evaluates a call's receiver (member lookup, then template + // substitution), which made an N-link fluent chain cost 2^N; the per-file + // step cap that used to hide that is gone. Evaluation never mutates scope + // or context, so within one outermost call a node's type cannot change; + // the memo is emptied when the next outermost call starts, because the + // resolver walk mutates scope between calls. + struct CLSPEvalMemoSlot *eval_memo; // open addressing, arena-owned + int *eval_memo_used; // slots filled by the current evaluation + int eval_memo_cap; // power of two, or 0 + int eval_memo_count; + CBMLSPStatus eval_failure; // sticky: stop refinement rather than repeat uncached work +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; // -1 = normal; 0 = fail the next memo allocation +#endif int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed } CLSPContext; +struct CLSPEvalMemoSlot { + const void *node_id; // TSNode.id; NULL = empty + const CBMType *result; +}; + // --- API --- void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int source_len, @@ -132,8 +153,28 @@ const CBMType *c_simplify_type(CLSPContext *ctx, const CBMType *t, bool unwrap_p void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin); +// A memo allocation/capacity failure is an explicit incomplete-refinement error. +#define CBM_C_LSP_DEPTH_ERROR "C/C++ expression recursion depth limit exceeded" +#define CBM_C_LSP_MEMO_ERROR "C/C++ expression memo allocation or capacity failure" +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +// Thread-local, scoped to memo allocations only; restore with -1 after a test. +void cbm_c_lsp_test_memo_fail_after(int successful_allocations); +void cbm_c_lsp_test_depth_fail(bool enabled); +uint32_t cbm_c_lsp_test_walks_take(void); +#endif + // Cross-file LSP: build registry from defs + stdlib, re-parse and resolve. -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +// Status variants distinguish memo/depth failure. Bool wrappers report success. +// Completed output before a failed walk is kept. +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, // NULL = parse internally @@ -146,7 +187,14 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in // Cross-file LSP using a pre-built shared registry (Tier 2). Skips the // per-file registry build; just parse + resolve. -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **include_paths, const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **include_paths, const char **include_ns_qns, @@ -178,7 +226,7 @@ typedef struct { // Process multiple C/C++ files' cross-file LSP in one CGo call. // out must point to file_count pre-zeroed CBMResolvedCallArray structs. -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out); #endif // CBM_LSP_C_LSP_H diff --git a/internal/cbm/lsp/kotlin_lsp.c b/internal/cbm/lsp/kotlin_lsp.c index 0ca49afd5a..c0e2f3febf 100644 --- a/internal/cbm/lsp/kotlin_lsp.c +++ b/internal/cbm/lsp/kotlin_lsp.c @@ -489,13 +489,8 @@ static const CBMScope *kt_scope_declaring_name(const CBMScope *scope, const char if (!name) return NULL; for (const CBMScope *current = scope; current; current = current->parent) { - for (const CBMScopeChunk *chunk = current->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return current; - } - } - } + if (cbm_scope_lookup_local(current, name)) + return current; } return NULL; } diff --git a/internal/cbm/lsp/lsp_node_iter.h b/internal/cbm/lsp/lsp_node_iter.h index 033962f14f..65db5c21b1 100644 --- a/internal/cbm/lsp/lsp_node_iter.h +++ b/internal/cbm/lsp/lsp_node_iter.h @@ -39,4 +39,33 @@ static inline TSNode* cbm_lsp_collect_children(CBMArena* arena, TSNode node, uin return kids; } +/* Named-children variant (#1527): the same single cursor pass, keeping only + * named nodes, so `for (i) ts_node_named_child(node, i)` over a wide node (a + * module holding thousands of classes, a body holding thousands of + * statements) becomes O(n). Returns NULL and sets *out_n=0 for a node without + * named children or on OOM; callers that must not lose children on OOM fall + * back to indexed access. */ +static inline TSNode *cbm_lsp_collect_named_children(CBMArena *arena, TSNode node, + uint32_t *out_n) { + uint32_t nc = ts_node_named_child_count(node); + *out_n = 0; + if (nc == 0) + return NULL; + TSNode *kids = (TSNode *)cbm_arena_alloc(arena, (size_t)nc * sizeof(TSNode)); + if (!kids) + return NULL; + uint32_t kn = 0; + TSTreeCursor cur = ts_tree_cursor_new(node); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode c = ts_tree_cursor_current_node(&cur); + if (ts_node_is_named(c)) + kids[kn++] = c; + } while (kn < nc && ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + *out_n = kn; + return kids; +} + #endif /* CBM_LSP_NODE_ITER_H */ diff --git a/internal/cbm/lsp/lsp_work.h b/internal/cbm/lsp/lsp_work.h new file mode 100644 index 0000000000..fa286293d3 --- /dev/null +++ b/internal/cbm/lsp/lsp_work.h @@ -0,0 +1,24 @@ +/* + * lsp_work.h — deterministic work counter for LSP complexity tests (#1527). + * + * Seam builds only (CBM_ENABLE_TEST_SEAMS): every name comparison a resolver + * primitive performs while searching (scope frame probes, enclosing-definition + * candidates) adds to a thread-local counter, so a test can assert that the + * work grows linearly with input size without reading a clock (O9). Product + * builds compile CBM_LSP_WORK to nothing. + */ +#ifndef CBM_LSP_WORK_H +#define CBM_LSP_WORK_H + +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +extern _Thread_local uint64_t cbm_lsp_work_steps; +#define CBM_LSP_WORK(n) (cbm_lsp_work_steps += (uint64_t)(n)) +/* Read and reset the calling thread's counter. */ +uint64_t cbm_lsp_work_take(void); +#else +#define CBM_LSP_WORK(n) ((void)0) +#endif + +#endif /* CBM_LSP_WORK_H */ diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 70de5dce78..e78bba7dcb 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -13,6 +13,7 @@ #include "py_lsp.h" #include "../cbm.h" #include "../helpers.h" +#include "lsp_node_iter.h" #include "tree_sitter/api.h" #include #include @@ -25,14 +26,80 @@ * only from py_lsp.c, never compiled standalone. */ #include "py_builtins.c" -/* Guards for py_eval_expr_type — mirrors c_eval_expr_type's guard design - * (C_EVAL_DEPTH_LIMIT / C_EVAL_MAX_STEPS_PER_FILE in c_lsp.c). */ +/* Stack guard for py_eval_expr_type (#720; same limit as C_EVAL_DEPTH_LIMIT). + * There is deliberately NO per-file work budget (#1527): the step cap that + * used to sit here stopped resolution after 10000 evaluations and left the + * rest of a large file silently unresolved while the file still reported + * status `indexed`. Memoized evaluation (#710), hashed scope frames, and + * cursor-based child access reduce the repeated work. Memo failure is reported + * explicitly; the existing depth bound now reports an incomplete walk too. */ #define PY_LSP_MAX_EVAL_DEPTH 256 -#define PY_EVAL_MAX_STEPS_PER_FILE 10000 +#include "lsp_work.h" + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int py_test_memo_allocations_left = -1; +static _Thread_local bool py_test_depth_failure; +void cbm_py_lsp_test_depth_fail(bool enabled) { + py_test_depth_failure = enabled; +} +void cbm_py_lsp_test_memo_fail_after(int successful_allocations) { + py_test_memo_allocations_left = successful_allocations; +} +#endif + +static void py_memo_test_stage(PyLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + const char *depth_stage = getenv("CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 256; + const char *fail_stage = getenv("CBM_TEST_PY_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime also works when the result arena cannot allocate. */ + result->error_msg = + status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR : CBM_PY_LSP_MEMO_ERROR; +} // Forward decls static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node); +/* Named children of one node for an index loop (#1527). ts_node_named_child(n, i) + * restarts tree-sitter's child iterator at the first child, so indexing every + * child of a wide node is O(n^2) -- a module of thousands of classes, or a body + * of thousands of statements, spent most of its resolve time re-walking its + * own children. Wide nodes are collected once (lsp_node_iter.h); narrow nodes + * keep plain indexed access, which is cheaper than an allocation. */ +enum { PY_KIDS_COLLECT_MIN = 64 }; +typedef struct { + TSNode node; + TSNode *kids; /* NULL = index the node directly */ + uint32_t n; +} PyKids; + +static PyKids py_kids(CBMArena *arena, TSNode node) { + PyKids k = {node, NULL, ts_node_named_child_count(node)}; + if (k.n >= PY_KIDS_COLLECT_MIN) { + uint32_t got = 0; + TSNode *kids = cbm_lsp_collect_named_children(arena, node, &got); + if (kids && got == k.n) + k.kids = kids; + } + return k; +} + +static TSNode py_kid(const PyKids *k, uint32_t i) { + return k->kids ? k->kids[i] : ts_node_named_child(k->node, i); +} + /* Decorators are extracted as raw syntax (`@property`, `@pkg.cache(...)`), * not resolved qualified names. Preserve the raw array for sound callable- * value admission, while parsing only enough of the spelling to retain the @@ -142,7 +209,7 @@ static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) { * The walk_depth-- runs after the inner returns, so early returns in the body * never leak the counter. */ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { - if (ctx->walk_depth >= cbm_lsp_max_walk_depth()) + if (!ctx || ctx->eval_failure || ctx->walk_depth >= cbm_lsp_max_walk_depth()) return; ctx->walk_depth++; py_resolve_calls_in_inner(ctx, node); @@ -229,6 +296,11 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou if (!ctx) return; memset(ctx, 0, sizeof(PyLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = py_test_memo_allocations_left; + if (py_test_depth_failure) + ctx->eval_depth = 256; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; @@ -505,16 +577,19 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod return py_import_match_result(&match, qn_io); } -static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local, - const char **qn_io) { +/* stmts: the root's import statements, collected once per root by the caller + * (#1527) -- only they can match, and rescanning every top-level statement + * once per import made classification O(imports x module size). */ +static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, bool have_root, + const TSNode *stmts, uint32_t stmt_count, + const char *local, const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; - if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root)) + if (!ctx || !local || !qn || !qn_io || !have_root) return PY_DIRECT_IMPORT_UNKNOWN; PyImportSyntaxMatch match = {0}; - uint32_t root_count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < root_count; i++) { - py_import_match_statement(ctx, ts_node_named_child(root, i), local, qn, &match); + for (uint32_t i = 0; i < stmt_count; i++) { + py_import_match_statement(ctx, stmts[i], local, qn, &match); } return py_import_match_result(&match, qn_io); } @@ -561,12 +636,32 @@ static void py_bind_dotted_prefixes(PyLSPContext *ctx, const char *qn) { static void py_classify_imports_for_root(PyLSPContext *ctx, TSNode root) { if (!ctx) return; + /* A null root classifies every import UNKNOWN, as it always has; so does + * a root whose statement list cannot be allocated (no proof either way). */ + bool have_root = !ts_node_is_null(root); + TSNode *stmts = NULL; + uint32_t stmt_count = 0; + if (have_root && ctx->import_count > 0) { + PyKids rk = py_kids(ctx->arena, root); + if (rk.n > 0) { + stmts = (TSNode *)cbm_arena_alloc(ctx->arena, (size_t)rk.n * sizeof(TSNode)); + if (!stmts) + have_root = false; + } + for (uint32_t i = 0; stmts && i < rk.n; i++) { + TSNode c = py_kid(&rk, i); + const char *ck = ts_node_type(c); + if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) + stmts[stmt_count++] = c; + } + } for (int i = 0; i < ctx->import_count; i++) { const char *local = ctx->import_local_names[i]; const char *qn = ctx->import_module_qns[i]; if (!local || !qn) continue; - PyDirectImportKind direct_kind = py_import_kind_from_ast(ctx, root, local, &qn); + PyDirectImportKind direct_kind = + py_import_kind_from_ast(ctx, have_root, stmts, stmt_count, local, &qn); ctx->import_module_qns[i] = qn; if (ctx->import_kinds) ctx->import_kinds[i] = (unsigned char)direct_kind; @@ -783,6 +878,8 @@ static const char *py_lookup_dict_dispatch(PyLSPContext *ctx, const char *var, c return NULL; } +enum { PY_RESOLVED_DEDUP_WINDOW = 256 }; + static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_qn, const char *strategy, float confidence, const char *reason, TSNode site) { @@ -796,9 +893,8 @@ static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_q // entries catches the common case while keeping per-emission O(1). // Without this cap the dedup is O(N) per emission -> O(N^2) per file // and dominates above ~1k call sites. - enum { DEDUP_WINDOW = 256 }; int n = ctx->resolved_calls->count; - int start = n > DEDUP_WINDOW ? n - DEDUP_WINDOW : 0; + int start = n > PY_RESOLVED_DEDUP_WINDOW ? n - PY_RESOLVED_DEDUP_WINDOW : 0; for (int i = start; i < n; i++) { CBMResolvedCall *rc = &ctx->resolved_calls->items[i]; if (rc->kind == CBM_RESOLVED_INVOCATION && rc->site_start_byte == site_start && @@ -1422,7 +1518,7 @@ static const CBMType *py_iterable_element_type(PyLSPContext *ctx, const CBMType } /* The real recursive-descent evaluator. Never call directly — go through - * the memoizing, depth- and budget-guarded py_eval_expr_type wrapper below + * the memoizing, depth-guarded py_eval_expr_type wrapper below * (every recursive call inside this body already does). */ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) @@ -2086,6 +2182,14 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { if (ctx->type_cache_cap >= PY_TYPE_CACHE_MAX_CAP) return false; int new_cap = ctx->type_cache_cap ? ctx->type_cache_cap * 2 : PY_TYPE_CACHE_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(CBMPyTypeCacheEntry)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return false; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif CBMPyTypeCacheEntry *new_entries = (CBMPyTypeCacheEntry *)cbm_arena_alloc( ctx->arena, (size_t)new_cap * sizeof(CBMPyTypeCacheEntry)); if (!new_entries) @@ -2111,9 +2215,9 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { return true; } -static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { +static bool py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { if (!id || !result) - return; + return true; /* Keep load strictly below 75% so probe chains stay short. If growth * fails (OOM / max cap), REJECT the insert rather than filling up: a * full table would turn every probe loop into a full-table scan, and @@ -2121,7 +2225,7 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp if (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3) { if (!py_type_cache_grow(ctx) && (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3)) - return; + return false; } uint32_t mask = (uint32_t)ctx->type_cache_cap - 1; uint32_t idx = py_type_cache_hash(id) & mask; @@ -2132,31 +2236,33 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp e->gen = ctx->type_cache_gen; e->result = result; ctx->type_cache_count++; - return; + return true; } if (e->node_id == id) { e->gen = ctx->type_cache_gen; /* refresh a stale-generation entry in place */ e->result = result; - return; + return true; } idx = (idx + 1) & mask; } /* Probe bound exhausted — only reachable with a corrupt count; drop the * insert instead of spinning. */ + return false; } -/* Memoizing, depth- and budget-guarded wrapper — the function every call - * site in this file goes through. */ +/* Memoizing, depth-guarded wrapper — the function every call site in this + * file goes through. */ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); /* Depth cap (issue #720): the evaluator recurses once per expression * nesting level, so a pathologically deep expression (tens of - * thousands of parens) overflowed the native stack. Same limit as - * C_EVAL_DEPTH_LIMIT. Past the cap: unknown, and NEVER cached. */ + * thousands of parens) overflowed the native stack. At the existing bound, + * stop this walk explicitly; never memoize a truncated result as complete. */ if (ctx->eval_depth >= PY_LSP_MAX_EVAL_DEPTH) { ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } @@ -2164,17 +2270,9 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { if (cached) return cached; - /* Per-file work budget (mirrors C_EVAL_MAX_STEPS_PER_FILE): expression - * type evaluation is best-effort, so pathological files degrade to - * unknown instead of stalling repository indexing. Only real - * evaluations consume budget — cache hits above are O(1). */ - if (ctx->eval_steps++ > PY_EVAL_MAX_STEPS_PER_FILE) { - ctx->eval_truncations++; - if (ctx->debug && ctx->eval_steps == PY_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [pylsp] expression eval step budget exhausted; returning unknown\n"); - } - return cbm_type_unknown(); - } + /* One real (non-memoized) evaluation: the work the complexity tests count + * to prove evaluation stays linear without a budget. */ + CBM_LSP_WORK(1); uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; @@ -2184,19 +2282,25 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { result = cbm_type_unknown(); /* Only cache full-fidelity results: if any descendant evaluation was - * cut off by the depth cap or the step budget, this result may be a - * truncated `unknown`. Caching it would poison later evaluations that - * reach the same node from a shallower frame (or with fresh budget) — - * exactly the reuse the guards are supposed to preserve. */ - if (ctx->eval_truncations == trunc_before) - py_type_cache_insert(ctx, node.id, result); - return result; + * cut off by the depth cap, this result may be a truncated `unknown`. + * The sticky failure stops this walk instead of repeating uncached branches; + * a fresh walk starts with a fresh cache. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !py_type_cache_insert(ctx, node.id, result)) + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + return ctx->eval_failure ? cbm_type_unknown() : result; } +#ifdef CBM_ENABLE_TEST_SEAMS +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node) { + return py_eval_expr_type(ctx, node); +} +#endif + /* ── statement processing: bind from assignments, for-loops, with-as ──── */ static void py_process_statement(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return; const char *k = ts_node_type(node); @@ -2502,6 +2606,8 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { /* ── Recursive walker: process statements + emit resolved_calls ── */ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { + if (!ctx || ctx->eval_failure) + return; if (!ctx || ctx->callable_value_proof_disabled) return; TSNode fn = ts_node_child_by_field_name(call_node, "function", 8); @@ -3443,9 +3549,9 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { return; } - uint32_t nc = ts_node_named_child_count(node); - for (uint32_t i = 0; i < nc; i++) { - py_resolve_calls_in(ctx, ts_node_named_child(node, i)); + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n; i++) { + py_resolve_calls_in(ctx, py_kid(&nk, i)); } } @@ -4006,6 +4112,8 @@ static void py_bind_parameters(PyLSPContext *ctx, TSNode params) { } static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char *container_qn) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(func_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4034,9 +4142,9 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char if (!ts_node_is_null(body)) { py_resolve_calls_in(ctx, body); // Also descend into nested function/class definitions in the body. - uint32_t bnc = ts_node_named_child_count(body); - for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + PyKids bk = py_kids(ctx->arena, body); + for (uint32_t i = 0; i < bk.n; i++) { + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, ctx->enclosing_func_qn); @@ -4064,6 +4172,8 @@ static bool py_is_init_method(PyLSPContext *ctx, TSNode func_node) { } static void py_process_class(PyLSPContext *ctx, TSNode class_node) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(class_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4076,13 +4186,14 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { TSNode body = ts_node_child_by_field_name(class_node, "body", 4); if (!ts_node_is_null(body)) { - uint32_t bnc = ts_node_named_child_count(body); + PyKids bk = py_kids(ctx->arena, body); + uint32_t bnc = bk.n; // First pass: process class-level annotated assignments (PEP 526 // class-body field annotations like `x: int`) and dunder __init__ // methods so fields are registered before sibling methods that // reference them. for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "expression_statement") == 0 && ts_node_named_child_count(c) > 0) { TSNode inner = ts_node_named_child(c, 0); @@ -4105,7 +4216,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Second pass: __init__ (so self.x assignments populate fields). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); TSNode fn_node = c; bool is_decorated = strcmp(ck, "decorated_definition") == 0; @@ -4123,7 +4234,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Third pass: every other method (and nested classes). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { if (!py_is_init_method(ctx, c)) { @@ -4148,10 +4259,17 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { /* Module bindings are replayed in source order before deferred function bodies * are analyzed. Class definitions always replace an earlier callable identity; * the registry decides whether the resulting class type is known precisely. */ -static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const char *name) { - uint32_t count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < count; i++) { - TSNode node = ts_node_named_child(root, i); +/* The names of the classes this source defines at module level, as a name set + * (a detached, hash-indexed CBMScope frame). Built in one pass over the root: + * asking "does the root define a class named X?" by rescanning the root once + * per registry type made module assembly O(types x top-level statements). + * Returns NULL if a class name or set entry cannot be collected completely. */ +static CBMScope *py_root_class_names(PyLSPContext *ctx, const PyKids *rk) { + CBMScope *names = cbm_scope_push(ctx->arena, NULL); + if (!names) + return NULL; + for (uint32_t i = 0; i < rk->n; i++) { + TSNode node = py_kid(rk, i); const char *kind = ts_node_type(node); if (strcmp(kind, "decorated_definition") == 0) { node = ts_node_child_by_field_name(node, "definition", 10); @@ -4159,20 +4277,33 @@ static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const ch } if (strcmp(kind, "class_definition") != 0) continue; - TSNode name_node = ts_node_child_by_field_name(node, "name", 4); - if (py_import_node_text_equals(ctx, name_node, name)) - return true; + char *name = py_node_text(ctx, ts_node_child_by_field_name(node, "name", 4)); +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_root_name_failure == 1) + name = NULL; + else if (ctx->test_root_name_failure == 2) + name = ""; +#endif + if (!name || !name[0] || !cbm_scope_bind_checked(names, name, cbm_type_unknown())) + return NULL; } - return false; + return names; } /* A project registry can contain another file's class in the same logical * Python module. Keep those cross-file globals available, but do not prebind * classes declared by this source: their binding epoch belongs in the ordered * replay below. */ -static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { +static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) { if (!ctx || !ctx->registry || !ctx->module_qn) return; + CBMScope *defined = py_root_class_names(ctx, rk); + if (!defined) { + /* Without the set no external class can be proven undeclared here; + * binding none is the fail-closed direction. */ + py_disable_callable_value_proof(ctx); + return; + } size_t prefix_len = strlen(ctx->module_qn); CBMTypeShortIter all_types; cbm_registry_all_types_chain(ctx->registry, &all_types); @@ -4181,13 +4312,20 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { const char *qn = type->qualified_name; const char *name = type->short_name; if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || - py_root_defines_class_named(ctx, root, name)) { + cbm_scope_lookup_local(defined, name)) { continue; } py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); } } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root) { + PyKids rk = py_kids(ctx->arena, root); + py_bind_external_module_classes(ctx, &rk); +} +#endif + static void py_bind_module_class(PyLSPContext *ctx, TSNode class_node) { if (!ctx || ts_node_is_null(class_node) || !ctx->module_qn) return; @@ -4564,9 +4702,9 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int } return; } - uint32_t count = ts_node_named_child_count(node); - for (uint32_t i = 0; i < count; i++) - py_invalidate_possible_bindings(ctx, ts_node_named_child(node, i), depth + 1); + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n; i++) + py_invalidate_possible_bindings(ctx, py_kid(&nk, i), depth + 1); } static bool py_replayable_import_kind(PyDirectImportKind kind) { @@ -4654,10 +4792,22 @@ static bool py_expression_is_annotation_only_assignment(TSNode statement) { } void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { - if (!ctx || ts_node_is_null(root)) + if (!ctx || ctx->eval_failure || ts_node_is_null(root)) return; + int resolved_start = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + int synthetic_start = ctx->syn_calls ? ctx->syn_calls->count : 0; + /* Only invocation dedup mutates prior rows, within this fixed tail. + * Snapshot once per walk: no full-output copy or fallible rollback allocation. */ + int tail_count = + resolved_start < PY_RESOLVED_DEDUP_WINDOW ? resolved_start : PY_RESOLVED_DEDUP_WINDOW; + int tail_start = resolved_start - tail_count; + CBMResolvedCall saved_tail[PY_RESOLVED_DEDUP_WINDOW]; + CBMResolvedCall *original_tail = tail_count ? ctx->resolved_calls->items + tail_start : NULL; + if (tail_count) + memcpy(saved_tail, original_tail, (size_t)tail_count * sizeof(*saved_tail)); + PyKids rk = py_kids(ctx->arena, root); py_classify_imports_for_root(ctx, root); - py_bind_external_module_classes(ctx, root); + py_bind_external_module_classes(ctx, &rk); unsigned char *consumed_imports = NULL; if (ctx->import_count > 0) { consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); @@ -4665,13 +4815,13 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { memset(consumed_imports, 0, (size_t)ctx->import_count); } - uint32_t nc = ts_node_named_child_count(root); + uint32_t nc = rk.n; const char *prev_func = ctx->enclosing_func_qn; ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.__module__", ctx->module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) { py_replay_import_statement(ctx, c, consumed_imports); @@ -4709,8 +4859,8 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } // Pass 2: top-level calls (rare) and nested definitions. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, NULL); @@ -4729,6 +4879,17 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } ctx->enclosing_func_qn = prev_func; + if (ctx->eval_failure) { + if (tail_count) { + memcpy(original_tail, saved_tail, (size_t)tail_count * sizeof(*saved_tail)); + memcpy(ctx->resolved_calls->items + tail_start, saved_tail, + (size_t)tail_count * sizeof(*saved_tail)); + } + if (ctx->resolved_calls) + ctx->resolved_calls->count = resolved_start; + if (ctx->syn_calls) + ctx->syn_calls->count = synthetic_start; + } } /* Register one definition into the registry. Returns true if recognized. */ @@ -4887,7 +5048,7 @@ static bool py_register_def(CBMArena *arena, CBMTypeRegistry *reg, CBMDefinition void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root) { - if (!arena || !result) + if (!arena || !result || result->lsp_skipped) return; /* Inject minimal builtin definitions as real graph nodes (builtins.len, @@ -4913,6 +5074,18 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, } py_mark_ambiguous_callable_bindings(®); + /* Hash-index the registry before the walk (#1527). Unfinalized, every + * lookup scanned the stdlib plus all of this file's definitions, which + * made resolving a large module O(calls x defs). The index answers each + * duplicated name with its first registration, as the scan did, so the + * resolution result is unchanged; post-walk additions (instance fields) + * stay visible through the registry's tail scan. The index lives in a + * scratch arena that dies with this call, not in the result arena. */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + PyLSPContext ctx; py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, &result->resolved_calls); /* Let the resolver inject synthetic syntactic calls for operator/subscript @@ -4926,7 +5099,11 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, } } + py_memo_test_stage(&ctx, "raw"); py_lsp_process_file(&ctx, root); + if (ctx.eval_failure) + cbm_py_lsp_record_failure(result, ctx.eval_failure); + cbm_arena_destroy(&idx_arena); } /* ── Cross-file + batch ───────────────────────────────────────── */ @@ -5094,13 +5271,23 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe } } -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_status(arena, source, source_len, module_qn, defs, def_count, + import_names, import_qns, import_count, cached_tree, out, + synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5108,13 +5295,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); @@ -5141,6 +5328,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); cbm_arena_destroy(&idx_arena); @@ -5148,6 +5336,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } /* ── Tier 2: pre-built per-language registry (mirrors Go pilot) ── */ @@ -5175,13 +5364,22 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i return reg; } -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, reg, import_names, import_qns, import_count, + cached_tree, out, synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5189,13 +5387,13 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); @@ -5208,19 +5406,22 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); if (owns_tree && tree) ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!arena || !files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchPyLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -5233,9 +5434,10 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ CBMResolvedCallArray file_out; memset(&file_out, 0, sizeof(file_out)); - cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->defs, file->def_count, file->import_names, file->import_qns, - file->import_count, file->cached_tree, &file_out, NULL); + if (!cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, + file->defs, file->def_count, file->import_names, file->import_qns, + file->import_count, file->cached_tree, &file_out, NULL)) + complete = false; if (file_out.count > 0) { out[f].count = file_out.count; @@ -5264,6 +5466,7 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ cbm_arena_destroy(&file_arena); } + return complete; } /* ── Stdlib stub — Phase 10 replaces with auto-generated body ─── */ diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 939025100f..e7536b8f9e 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -95,14 +95,18 @@ typedef struct { // py_eval_expr_type memoization + guards (issues #710/#720; mirrors // c_eval_expr_type's guard design in c_lsp.c). All zero via memset — - // each file starts with a cold cache and a full budget. + // each file starts with a cold cache. CBMPyTypeCacheEntry *type_cache; // open addressing, linear probe, arena-allocated int type_cache_count; // occupied slots (kept < 75% of cap) int type_cache_cap; // power-of-two capacity uint32_t type_cache_gen; // bumped on every scope mutation (O(1) flush) int eval_depth; // evaluator recursion depth (PY_LSP_MAX_EVAL_DEPTH) - int eval_steps; // per-file work budget used (PY_EVAL_MAX_STEPS_PER_FILE) - uint32_t eval_truncations; // depth/budget cutoff count — gates memo inserts + uint32_t eval_truncations; // depth-cap cutoff count — gates memo inserts + CBMLSPStatus eval_failure; // sticky: memo/depth failure makes this walk incomplete +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; + int test_root_name_failure; // 1 = NULL, 2 = empty; only the root-name collection +#endif // Sticky fail-closed guard for exact callable-value proof. If a required // scope/name/overlay allocation fails, later semantic passes must leave @@ -161,9 +165,28 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, // Phase 10 fills the body; Phase 2 ships a no-op so linkage works. void cbm_python_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); +#define CBM_PY_LSP_DEPTH_ERROR "Python expression recursion depth limit exceeded" +#define CBM_PY_LSP_MEMO_ERROR "Python expression memo allocation or capacity failure" +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_memo_fail_after(int successful_allocations); +void cbm_py_lsp_test_depth_fail(bool enabled); +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root); +#endif + // --- Cross-file LSP resolution --- +// Status variants distinguish memo/depth failure; bool wrappers report success. +// Existing no-op/parser exits retain their previous behavior. + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, // NULL = parse internally @@ -173,7 +196,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, * Filters all_defs to Python entries, builds + finalizes once. */ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count); -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); + +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **import_names, const char **import_qns, @@ -194,7 +223,7 @@ typedef struct { int import_count; } CBMBatchPyLSPFile; -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out); #endif // CBM_LSP_PY_LSP_H diff --git a/internal/cbm/lsp/scope.c b/internal/cbm/lsp/scope.c index 2d9740db0c..0b3c6505cb 100644 --- a/internal/cbm/lsp/scope.c +++ b/internal/cbm/lsp/scope.c @@ -1,5 +1,21 @@ #include "scope.h" +#include "lsp_work.h" #include +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +_Thread_local uint64_t cbm_lsp_work_steps = 0; +uint64_t cbm_lsp_work_take(void) { + uint64_t v = cbm_lsp_work_steps; + cbm_lsp_work_steps = 0; + return v; +} +#endif + +enum { + SCOPE_INDEX_MIN_CAP = 64, /* first table: > 2x one chunk, a power of two */ + SCOPE_INDEX_GROW = 2, +}; CBMScope* cbm_scope_push(CBMArena* a, CBMScope* current) { CBMScope* scope = (CBMScope*)cbm_arena_alloc(a, sizeof(CBMScope)); @@ -33,6 +49,114 @@ static CBMScopeChunk* alloc_chunk(CBMScope* scope) { return c; } +static uint32_t scope_name_hash(const char *name) { + uint32_t h = 2166136261u; /* FNV-1a */ + for (const unsigned char *p = (const unsigned char *)name; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Place one binding into a table that has room (load kept below 1/2). */ +static void scope_index_place(CBMVarBinding **slots, int cap, CBMVarBinding *b) { + uint32_t mask = (uint32_t)cap - 1; + uint32_t i = scope_name_hash(b->name) & mask; + while (slots[i]) { + i = (i + 1) & mask; + } + slots[i] = b; +} + +/* (Re)build the frame's table at cap slots from its chunks. On allocation + * failure the frame drops its index and stays on the always-correct scan. */ +static void scope_index_rebuild(CBMScope *scope, int cap) { + /* Recovery can start with many bindings and no index. Always size from + * the complete frame, never just from the previous table's capacity. */ + if (scope->binding_count < 0 || scope->binding_count > INT_MAX / 2) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + if (cap < SCOPE_INDEX_MIN_CAP) + cap = SCOPE_INDEX_MIN_CAP; + while (cap < scope->binding_count * 2) { + if (cap > INT_MAX / SCOPE_INDEX_GROW) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + cap *= SCOPE_INDEX_GROW; + } + if ((size_t)cap > SIZE_MAX / sizeof(CBMVarBinding *)) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + CBMVarBinding **slots = + scope->arena + ? (CBMVarBinding **)cbm_arena_alloc(scope->arena, (size_t)cap * sizeof(CBMVarBinding *)) + : NULL; + if (!slots) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + memset(slots, 0, (size_t)cap * sizeof(CBMVarBinding *)); + int count = 0; + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + if (c->bindings[i].name) { + scope_index_place(slots, cap, &c->bindings[i]); + count++; + } + } + } + scope->index.slots = slots; + scope->index.cap = cap; + scope->index.count = count; +} + +/* Record a freshly appended binding in the frame's index, building the index + * the first time the frame outgrows one chunk. */ +static void scope_index_note(CBMScope *scope, CBMVarBinding *b) { + if (scope->index.cap == 0) { + if (scope->binding_count > CBM_SCOPE_CHUNK_BINDINGS) { + scope_index_rebuild(scope, SCOPE_INDEX_MIN_CAP); + } + return; + } + if (scope->index.count >= scope->index.cap / 2) { + scope_index_rebuild(scope, scope->index.cap); /* includes b; sizes from binding_count */ + return; + } + scope_index_place(scope->index.slots, scope->index.cap, b); + scope->index.count++; +} + +/* The binding of name in one frame: hashed when the frame has an index, + * otherwise the linear chunk scan (small frames, or no index memory). */ +static CBMVarBinding *scope_frame_find(const CBMScope *scope, const char *name) { + if (scope->index.cap > 0) { + uint32_t mask = (uint32_t)scope->index.cap - 1; + for (uint32_t i = scope_name_hash(name) & mask;; i = (i + 1) & mask) { + CBMVarBinding *b = scope->index.slots[i]; + if (!b) { + return NULL; + } + CBM_LSP_WORK(1); + if (strcmp(b->name, name) == 0) { + return b; + } + } + } + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + CBM_LSP_WORK(1); + if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { + return &c->bindings[i]; + } + } + } + return NULL; +} + /* Returns false when the binding could NOT be recorded in THIS frame. * * The failure that matters is arena exhaustion in alloc_chunk: the old void @@ -47,14 +171,11 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp if (!scope || !name) { return false; } - for (CBMScopeChunk* c = scope->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].type = type; - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *existing = scope_frame_find(scope, name); + if (existing) { + existing->type = type; + existing->callable_qn = callable_qn; + return true; } CBMScopeChunk* head = scope->chunks; if (!head || head->used >= CBM_SCOPE_CHUNK_BINDINGS) { @@ -63,10 +184,13 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp return false; /* arena exhausted: the shadow did NOT take effect */ } } - head->bindings[head->used].name = name; - head->bindings[head->used].type = type; - head->bindings[head->used].callable_qn = callable_qn; + CBMVarBinding *b = &head->bindings[head->used]; + b->name = name; + b->type = type; + b->callable_qn = callable_qn; head->used++; + scope->binding_count++; + scope_index_note(scope, b); return true; } @@ -88,6 +212,13 @@ bool cbm_scope_bind_callable_checked(CBMScope *scope, const char *name, const CB return cbm_scope_bind_value(scope, name, type, callable_qn); } +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name) { + if (!scope || !name) { + return NULL; + } + return scope_frame_find(scope, name); +} + const CBMType* cbm_scope_lookup(const CBMScope* scope, const char* name) { const CBMVarBinding *binding = cbm_scope_lookup_binding(scope, name); if (binding) @@ -109,13 +240,10 @@ bool cbm_scope_update_callable(CBMScope *scope, const char *name, const char *ca return false; } for (CBMScope *s = scope; s != NULL; s = s->parent) { - for (CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *b = scope_frame_find(s, name); + if (b) { + b->callable_qn = callable_qn; + return true; } } return false; diff --git a/internal/cbm/lsp/scope.h b/internal/cbm/lsp/scope.h index 5ba7265a36..af14cb7cd2 100644 --- a/internal/cbm/lsp/scope.h +++ b/internal/cbm/lsp/scope.h @@ -25,27 +25,48 @@ typedef struct CBMScopeChunk { struct CBMScopeChunk* next; } CBMScopeChunk; +/* Name index for one frame (#1527). A frame that outgrows one chunk (a module + * scope binding thousands of classes and functions) is looked up through an + * open-addressing table of binding pointers instead of a linear chunk scan -- + * the scan made every identifier lookup O(frame size) and a whole file + * O(n^2). Bindings never move (chunks are arena-allocated and only + * prepended) and a name is bound at most once per frame (rebinding updates + * in place), so the table maps each name to exactly the binding the linear + * scan would have found. The table is an accelerator only: when it cannot be + * allocated or grown the frame drops it and every lookup falls back to the + * scan, which is always correct. */ +typedef struct { + CBMVarBinding **slots; // arena-owned; NULL = empty + int cap; // power of two, or 0 = no index (linear scan) + int count; +} CBMScopeIndex; + typedef struct CBMScope { struct CBMScope* parent; CBMScopeChunk* chunks; CBMArena* arena; // owning arena, propagated to children at push time + int binding_count; // bindings recorded in THIS frame + CBMScopeIndex index; // built once binding_count exceeds one chunk } CBMScope; +/* The binding of name in THIS frame only (no parent walk), or NULL. */ +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name); + /* Return the complete nearest binding in one scope-chain walk, or NULL when * unbound. Keep this internal hot-path primitive inline: all language * resolvers use cbm_scope_lookup(), while Python also consumes the complete - * record to avoid repeating the same linear scan. */ + * record to avoid repeating the same walk. Each frame is probed through + * cbm_scope_lookup_local(), which uses the frame's hashed name index once the + * frame outgrows one chunk (#1527), so a lookup costs one probe per frame + * rather than a scan of every binding in scope. */ static inline const CBMVarBinding *cbm_scope_lookup_binding(const CBMScope *scope, const char *name) { if (!name) return NULL; for (const CBMScope *s = scope; s != NULL; s = s->parent) { - for (const CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) - return &c->bindings[i]; - } - } + const CBMVarBinding *b = cbm_scope_lookup_local(s, name); + if (b) + return b; } return NULL; } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 7fe5055fd2..a78c7433db 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -1391,12 +1391,8 @@ static const CBMScope *ts_scope_binding_owner(const CBMScope *scope, const char return NULL; } for (const CBMScope *s = scope; s; s = s->parent) { - for (const CBMScopeChunk *chunk = s->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return s; - } - } + if (cbm_scope_lookup_local(s, name)) { + return s; } } return NULL; diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 06b55c391d..216c347794 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -1,4 +1,5 @@ #include "type_registry.h" +#include "lsp_work.h" #include #include @@ -56,7 +57,12 @@ static void build_qn_index(CBMTypeRegistry *reg, CBMArena *idx_arena, bool for_f for (int i = 0; i < bucket_count; i++) buckets[i] = -1; - for (int i = 0; i < count; i++) { + /* Chains are built by prepending, so the entry inserted LAST is found + * first. Insert descending when the registry asked for first-registered + * answers (see index_first_registered). */ + bool first = reg->index_first_registered; + for (int k = 0; k < count; k++) { + int i = first ? count - 1 - k : k; const char *qn = for_funcs ? reg->funcs[i].qualified_name : reg->types[i].qualified_name; if (!qn) continue; @@ -96,7 +102,9 @@ static void build_method_index(CBMTypeRegistry *reg, CBMArena *idx_arena) { buckets[i] = -1; int idx = 0; - for (int i = 0; i < reg->func_count; i++) { + bool first = reg->index_first_registered; /* see build_qn_index */ + for (int k = 0; k < reg->func_count; k++) { + int i = first ? reg->func_count - 1 - k : k; const CBMRegisteredFunc *f = ®->funcs[i]; if (!f->receiver_type || !f->short_name) continue; @@ -779,6 +787,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->method_bucket_count - 1)); for (int idx = reg->method_buckets[slot]; idx >= 0; idx = reg->method_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->method_entries[idx].hash != h) continue; const CBMRegisteredFunc *f = ®->funcs[reg->method_entries[idx].payload_index]; @@ -792,6 +801,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, * post-finalize additions stay visible instead of silently vanishing. */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -802,6 +812,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, for (int i = 0; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -829,6 +840,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->type_qn_bucket_count - 1)); for (int idx = reg->type_qn_buckets[slot]; idx >= 0; idx = reg->type_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->type_qn_entries[idx].hash != h) continue; int p = reg->type_qn_entries[idx].payload_index; @@ -839,6 +851,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } /* Tail-scan types added after finalize (see lookup_method_self). */ for (int i = reg->type_qn_entry_count; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (reg->types[i].qualified_name && strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; @@ -848,6 +861,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; } @@ -873,6 +887,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->func_qn_bucket_count - 1)); for (int idx = reg->func_qn_buckets[slot]; idx >= 0; idx = reg->func_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->func_qn_entries[idx].hash != h) continue; int p = reg->func_qn_entries[idx].payload_index; @@ -883,6 +898,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } /* Tail-scan funcs added after finalize (see lookup_method_self). */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (reg->funcs[i].qualified_name && strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; @@ -892,6 +908,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; } diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index c752350b17..1ee284bf7c 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -143,6 +143,13 @@ typedef struct CBMTypeRegistry { * every lookup linear-scans it -> O(files*defs) (the Linux-kernel full-index * hang) plus a heap data race across workers. */ bool read_only; + // Set before finalize by a caller whose lookups used to run unfinalized + // (#1527): the exact-QN and method indexes then chain entries in + // registration order, so a duplicated name resolves to its FIRST + // registration -- the answer the linear scan gave. Without it the + // hashed chains answer with the LAST registration (the Tier-2 behavior, + // left unchanged). + bool index_first_registered; } CBMTypeRegistry; // Initialize a registry. diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 66751a6829..6629956e7e 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -1431,14 +1431,22 @@ void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source, int * imports — the existing pipeline doesn't carry C-style include * resolution as a separate map, so pass NULL/0 and let the LSP * fall back to its own #include scan. */ - cbm_run_c_lsp_cross(&scratch, source, source_len, module_qn, cpp_mode, defs, def_count, - NULL, NULL, 0, tree, &out); + { + CBMLSPStatus status = + cbm_run_c_lsp_cross_status(&scratch, source, source_len, module_qn, cpp_mode, defs, + def_count, NULL, NULL, 0, tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(r, status); + } break; } - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out, &synthetic_calls); - break; + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_status(&scratch, source, source_len, module_qn, + defs, def_count, imp_names, imp_qns, + imp_count, tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(r, status); + } break; case CBM_LANG_PHP: cbm_run_php_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out); @@ -1586,18 +1594,24 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * &out); used_prebuilt = true; break; - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, - imp_keys, imp_vals, imp_count, result->cached_tree, - &out, &synthetic_calls); + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, &overlay, imp_keys, imp_vals, imp_count, + result->cached_tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_C: case CBM_LANG_CPP: - case CBM_LANG_CUDA: - cbm_run_c_lsp_cross_with_registry(&scratch, source, source_len, def_module, - (lang != CBM_LANG_C), &overlay, imp_keys, imp_vals, - imp_count, result->cached_tree, &out); + case CBM_LANG_CUDA: { + CBMLSPStatus status = cbm_run_c_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, (lang != CBM_LANG_C), &overlay, imp_keys, + imp_vals, imp_count, result->cached_tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_CSHARP: @@ -1860,10 +1874,14 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * * file, not to a stale extraction marker (the innocent-quarantine * failure mode). */ cbm_index_mark_start(files[i].rel_path); + bool had_error = cache[i]->has_error; cbm_pxc_dispatch_file(lang, cache[i], source, source_len, files[i].rel_path, def_modules[i], &cross_registries, module_def_index, all_defs, def_count, imp_keys, imp_vals, imp_count, NULL, NULL); cbm_index_mark_done(files[i].rel_path); + if (!had_error && cache[i]->has_error && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, cache[i]->error_msg, + "lsp_skipped"); per_lang_calls++; processed++; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a91acaf831..726199fde8 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -75,6 +75,8 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" #include "lsp/rust_cargo.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ #include "pipeline/worker_pool.h" #include "foundation/compat.h" @@ -1292,12 +1294,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) { snprintf(how_far, sizeof(how_far), "%u/%u nodes walked", result->walk_nodes_visited, result->tree_nodes); pp_err_add(errs, fi->rel_path, how_far, "walk_truncated"); - } else if (result->lsp_skipped) { - /* Indexed, but without the per-file and cross-file LSP refinement: - * the same kind of hole from the other direction. Nothing in - * production sets this any more except a truncated walk (handled - * above) and the test seam — it is reported anyway, so that if - * something sets it again the gap arrives named, not silent. */ + } else if (result->lsp_skipped && !result->has_error) { + /* has_error already recorded its precise cause above. Indexed, but without the per-file + * and cross-file LSP refinement: the same kind of hole from the other direction. + * Nothing in production sets this any more except a truncated walk (handled above) and + * the test seam — it is reported anyway, so that if something sets it again the gap + * arrives named, not silent. */ char size_text[CBM_SZ_64]; snprintf(size_text, sizeof(size_text), "%u nodes", result->tree_nodes); pp_err_add(errs, fi->rel_path, size_text, "lsp_skipped"); @@ -1916,6 +1918,9 @@ typedef struct { /* Counters for parallel.resolve.lsp_cross_done summary. */ _Atomic int lsp_cross_processed; _Atomic int lsp_cross_skipped_no_source; + /* Each file is owned by one worker. The coordinator reports failures + * after join, without mutating the pipeline's error list concurrently. */ + const char **lsp_failures; /* Per-sub-phase timing (ns aggregated across workers) — surfaces * exactly where parallel_resolve's wall time is spent so we stop @@ -3748,11 +3753,20 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * file around the resolve so a hang HERE is attributed to * this file, not to a stale extraction marker. */ cbm_index_mark_start(rel); + bool had_error = result->has_error; cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, rc->cross_registries, rc->module_def_index, rc->all_defs, rc->def_count, imp_keys, imp_vals, imp_count, pp_rust_shared_registry_get, rc); cbm_index_mark_done(rel); + if (!had_error && result->has_error && result->error_msg && + (strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0)) { + rc->lsp_failures[file_idx] = result->error_msg; + ws->errors++; + } /* Free the on-demand re-read (no-op when source was retained). */ free_source(lsp_source_owned); /* Contract: cbm_slab_reclaim() requires the thread parser to be @@ -3865,6 +3879,11 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return CBM_NOT_FOUND; } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); + const char **lsp_failures = calloc((size_t)file_count, sizeof(*lsp_failures)); + if (!lsp_failures) { + cbm_aligned_free(workers); + return CBM_NOT_FOUND; + } bool have_rust = false; for (int i = 0; i < file_count; i++) { @@ -3905,6 +3924,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .module_def_index = module_def_index, .cross_registries = cross_registries, .rust_manifest = rust_manifest_ptr, + .lsp_failures = lsp_failures, }; atomic_init(&rc.next_file_idx, 0); atomic_init(&rc.lsp_cross_processed, 0); @@ -3922,6 +3942,12 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_scale_end(&rc.scale); CBM_PROF_END_N("parallel_resolve", "1_dispatch_workers_parallel", t_resolve_dispatch, file_count); + for (int i = 0; i < file_count; i++) { + if (lsp_failures[i] && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], + "lsp_skipped"); + } + free(lsp_failures); /* Workers joined: the shared Rust registry (if built) is no longer read. * Free its dedicated arena + the lock (registry was self-contained: it strdup'd * all QNs, so freeing all_defs afterward is safe). */ diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index d890911506..dbbef1d850 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -19,6 +19,7 @@ * - DLL patterns, SFINAE, placement new */ #include "test_framework.h" +#include "../src/foundation/compat.h" #include "cbm.h" #include "lsp/c_lsp.h" #include "lsp/py_lsp.h" @@ -26,7 +27,9 @@ #include "lsp/ts_lsp.h" #include "lsp/go_lsp.h" #include "lsp/type_registry.h" +#include "lsp/lsp_work.h" #include "../src/pipeline/lsp_resolve.h" +#include "../src/pipeline/pass_lsp_cross.h" #include "arena.h" #include "preprocessor.h" #include @@ -16499,7 +16502,479 @@ TEST(clsp_preprocessed_destructor_rewrite_respects_origin_during_rewrite) { PASS(); } +/* ── #1527: no per-file eval budget; cost fixed at the root ────────────── */ + +/* Resolved rows that name a real target (not an lsp_unresolved marker) whose + * callee QN contains sub. */ +static int count_real_resolved(const CBMFileResult *r, const char *sub) { + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + if (rc->callee_qn && strstr(rc->callee_qn, sub) && rc->confidence > 0.0f && + !(rc->strategy && strcmp(rc->strategy, "lsp_unresolved") == 0)) + n++; + } + return n; +} + +/* A versionbits-style fluent chain (the #323 hang): every link returns + * Tester&, and the evaluator re-evaluates a call's receiver for template + * substitution, so without a memo an N-link chain costs 2^N evaluations. The + * per-file step cap that used to contain it is gone; the per-evaluation memo + * must keep the work polynomial AND every link must resolve. */ +static char *clsp_chain_source(int links, int stmts) { + size_t sz = 512 + (size_t)stmts * (32 + (size_t)links * 40); + char *src = malloc(sz); + if (!src) + return NULL; + size_t pos = (size_t)snprintf(src, sz, + "struct Tester {\n" + " Tester& Mine(int h, int t, int v);\n" + " Tester& TestDefined();\n" + " Tester& TestStateSinceHeight(int h);\n" + " Tester& Reset();\n" + "};\n" + "void run() {\n" + " Tester t;\n"); + for (int s = 0; s < stmts; s++) { + pos += (size_t)snprintf(src + pos, sz - pos, " t.Reset()"); + for (int l = 0; l < links; l++) { + if (l % 3 == 0) + pos += (size_t)snprintf(src + pos, sz - pos, ".Mine(%d, %d, 0x100)", l, l); + else if (l % 3 == 1) + pos += (size_t)snprintf(src + pos, sz - pos, ".TestDefined()"); + else + pos += (size_t)snprintf(src + pos, sz - pos, ".TestStateSinceHeight(%d)", l); + } + pos += (size_t)snprintf(src + pos, sz - pos, ";\n"); + } + snprintf(src + pos, sz - pos, "}\n"); + return src; +} + +TEST(clsp_issue1527_fluent_chain_work_is_polynomial) { + enum { LINKS = 14, STMTS = 3 }; + char *src = clsp_chain_source(LINKS, STMTS); + ASSERT_NOT_NULL(src); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + ASSERT_NOT_NULL(r); + printf(" chain %dx%d: work=%llu\n", LINKS, STMTS, (unsigned long long)work); + /* Every link of every statement resolves to its Tester method. */ + ASSERT_EQ(count_real_resolved(r, ".Tester."), STMTS * (LINKS + 1)); + /* Memoized: ~LINKS^2 evaluations per statement (each call node starts a + * fresh outermost evaluation of its receiver chain) -- ~1.5k in total. + * Unmemoized: ~2^LINKS per call node, ~100k in total. */ + ASSERT(work < 10000); + cbm_free_result(r); + PASS(); +} + +/* Deterministic complexity gate for the C/C++ resolver (O9: no clock): 4x the + * definitions and calls must cost ~4x the counted work (scope-frame probes, + * registry lookup probes, enclosing-definition candidates, real + * evaluations). Methods are DEFINED (out of class), so they are registered + * before the registry is indexed. Declaration-only methods are registered by + * the walk itself, after indexing, and are still found by a linear tail scan + * -- a separate, known quadratic (see the #1527 follow-up), not measured here. */ +static uint64_t clsp_member_calls_work(int n, int *out_real) { + size_t sz = (size_t)n * 128 + 512; + char *src = malloc(sz); + if (!src) + return 0; + size_t pos = (size_t)snprintf(src, sz, "struct S {\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " S& m%d();\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "};\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, "S& S::m%d() { return *this; }\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "void use() {\n S s;\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " s.m%d().m%d();\n", i, (i + 1) % n); + snprintf(src + pos, sz - pos, "}\n"); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + *out_real = r ? count_real_resolved(r, ".S.m") : -1; + if (r) + cbm_free_result(r); + return work; +} + +TEST(clsp_issue1527_scale_work_is_linear) { + int real1 = 0, real4 = 0; + uint64_t w1 = clsp_member_calls_work(1000, &real1); + uint64_t w4 = clsp_member_calls_work(4000, &real4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (real=%d) 4000=%llu (real=%d) ratio=%.2fx\n", + (unsigned long long)w1, real1, (unsigned long long)w4, real4, ratio); + ASSERT_EQ(real1, 2 * 1000); + ASSERT_EQ(real4, 2 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +/* A reachable post-allocation-failure state, pinned without memory pressure. + * The old recovery path returned a full table here. Check capacity BEFORE + * any missing-name lookup, and free the arena before the predicted RED. */ +TEST(scope_issue1527_recovery_sizes_from_all_bindings) { + CBMArena a; + cbm_arena_init(&a); + CBMScope *scope = cbm_scope_push(&a, NULL); + char names[64][24]; + bool setup_ok = scope != NULL; + for (int i = 0; i < 63 && setup_ok; i++) { + snprintf(names[i], sizeof(names[i]), "binding_%d", i); + setup_ok = cbm_scope_bind_callable_checked(scope, names[i], cbm_type_unknown(), "old"); + } + const CBMVarBinding *stable = setup_ok ? cbm_scope_lookup_local(scope, names[7]) : NULL; + if (setup_ok) + memset(&scope->index, 0, sizeof(scope->index)); + snprintf(names[63], sizeof(names[63]), "binding_63"); + bool bound = + setup_ok && cbm_scope_bind_callable_checked(scope, names[63], cbm_type_unknown(), "old"); + bool spare = bound && scope->binding_count == 64 && scope->index.cap >= 128; + bool preserved = false; + if (spare) { + preserved = cbm_scope_lookup_local(scope, "absent") == NULL; + for (int i = 0; i < 64; i++) + preserved = preserved && cbm_scope_lookup_local(scope, names[i]) != NULL; + preserved = preserved && + cbm_scope_bind_callable_checked(scope, names[7], cbm_type_unknown(), "new") && + cbm_scope_lookup_local(scope, names[7]) == stable && stable->callable_qn && + strcmp(stable->callable_qn, "new") == 0 && scope->binding_count == 64; + CBMScope *child = cbm_scope_push(&a, scope); + preserved = preserved && child && + cbm_scope_bind_callable_checked(child, names[7], cbm_type_unknown(), "child"); + const CBMVarBinding *shadow = child ? cbm_scope_lookup_binding(child, names[7]) : NULL; + preserved = preserved && shadow && strcmp(shadow->callable_qn, "child") == 0 && + strcmp(stable->callable_qn, "new") == 0; + } + cbm_arena_destroy(&a); + ASSERT_TRUE(setup_ok); + ASSERT_TRUE(bound); + ASSERT_TRUE(spare); + ASSERT_TRUE(preserved); + PASS(); +} + +/* Initial allocation, second allocation, and later growth fail independently + * of the arena. The failure must be explicit, not an uncached exponential walk. */ +TEST(clsp_issue1527_memo_allocation_failure_is_reported) { + char *src = clsp_chain_source(40, 1); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int successful_allocations = 0; successful_allocations <= 2; successful_allocations++) { + (void)cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(successful_allocations); + CBMFileResult *r = extract_cpp(src); + uint32_t walks = cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0 && walks == 1; + if (r) { + /* The allocation-free diagnostic also survives result compaction. */ + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && !fresh->lsp_skipped && + count_real_resolved(fresh, ".Tester.") == 41; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +extern const TSLanguage *tree_sitter_cpp(void); + +TEST(clsp_issue1527_memo_and_depth_failures_are_sticky) { + const char *src = "void f() { x; }"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_cpp()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode fn = ts_node_named_child(ts_tree_root_node(tree), 0); + TSNode body = ts_node_child_by_field_name(fn, "body", 4); + TSNode expr = ts_node_named_child(ts_node_named_child(body, 0), 0); + bool identifier = !ts_node_is_null(expr) && strcmp(ts_node_type(expr), "identifier") == 0; + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMResolvedCallArray out = {0}; + CLSPContext ctx; + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + const CBMType *first = cbm_type_named(&a, "test.First"); + const CBMType *second = cbm_type_named(&a, "test.Second"); + cbm_scope_bind(ctx.current_scope, "x", first); + ctx.eval_depth = 257; + bool depth_unknown = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A depth-truncated walk is incomplete; only a fresh walk may resume. */ + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + cbm_scope_bind(ctx.current_scope, "x", first); + bool shallow_ok = c_eval_expr_type(&ctx, expr) == first && !ctx.eval_failure; + cbm_scope_bind(ctx.current_scope, "x", second); + bool rebound = c_eval_expr_type(&ctx, expr) == second; + /* Force a new memo's allocation to fail; old slots remain arena-owned. */ + ctx.eval_memo_cap = ctx.eval_memo_count = 0; + ctx.eval_memo = NULL; + ctx.eval_memo_used = NULL; + ctx.test_memo_allocations_left = 0; + bool failed = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + bool stopped = true; + for (int i = 0; i < 20; i++) + stopped = stopped && cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + uint64_t later_work = cbm_lsp_work_take(); + cbm_arena_destroy(&a); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(identifier); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(shallow_ok); + ASSERT_TRUE(rebound); + ASSERT_TRUE(failed); + ASSERT_TRUE(stopped); + ASSERT_EQ(later_work, 0); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_memo_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_MEMO_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_depth_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_cross_file_memo_failures_propagate) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + bool failed[3] = {false}, recovered[3] = {false}; + bool shared_ready = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMTypeRegistry *shared = cbm_c_build_cross_registry(&a, defs, 2); + shared_ready = shared_ready && shared && shared->read_only; + reg.fallback = shared; /* the production prebuilt path uses a writable overlay */ + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .cpp_mode = true, + .defs = defs, + .def_count = 2}; + cbm_c_lsp_test_memo_fail_after(0); + bool ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, + NULL, NULL, 0, NULL, &out) + : mode == 1 + ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", true, + ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + cbm_c_lsp_test_memo_fail_after(-1); + failed[mode] = !ok && out.count == 0; + ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, NULL, + NULL, 0, NULL, &out) + : mode == 1 ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", + true, ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + recovered[mode] = ok && out.count > 0; + cbm_arena_destroy(&a); + } + ASSERT_TRUE(shared_ready); + for (int i = 0; i < 3; i++) { + ASSERT_TRUE(failed[i]); + ASSERT_TRUE(recovered[i]); + } + PASS(); +} + +TEST(registry_issue1527_index_preserves_registration_order) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMRegisteredType t = {.qualified_name = "test.S", .short_name = "S"}; + cbm_registry_add_type(®, t); + cbm_registry_add_type(®, t); + CBMRegisteredFunc f = { + .qualified_name = "test.S.m", .short_name = "m", .receiver_type = "test.S"}; + cbm_registry_add_func(®, f); + cbm_registry_add_func(®, f); + bool linear_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = true; + cbm_registry_finalize(®); + bool indexed_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = false; + cbm_registry_finalize(®); + bool default_last = cbm_registry_lookup_type(®, "test.S") == ®.types[1] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[1] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[1]; + f.qualified_name = "test.S.tail"; + f.short_name = "tail"; + cbm_registry_add_func(®, f); + bool tail = cbm_registry_lookup_func(®, "test.S.tail") == ®.funcs[2] && + cbm_registry_lookup_method(®, "test.S", "tail") == ®.funcs[2]; + cbm_arena_destroy(&a); + ASSERT_TRUE(linear_first); + ASSERT_TRUE(indexed_first); + ASSERT_TRUE(default_last); + ASSERT_TRUE(tail); + PASS(); +} + +TEST(clsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + cbm_c_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_cpp(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_c_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2, + .cpp_mode = true}; + if (mode == 2) { + correct = !cbm_batch_c_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_c_lsp_cross_status(&arena, src, (int)strlen(src), "test", true, + defs, 2, NULL, NULL, 0, NULL, &out) + : cbm_run_c_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", true, &overlay, NULL, + NULL, 0, NULL, &out); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_CPP, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_c_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(c_lsp) { + RUN_TEST(clsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(scope_issue1527_recovery_sizes_from_all_bindings); + RUN_TEST(clsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(clsp_issue1527_memo_and_depth_failures_are_sticky); + RUN_TEST(clsp_issue1527_preprocessed_memo_failure_is_reported); + RUN_TEST(clsp_issue1527_preprocessed_depth_failure_is_reported); + RUN_TEST(registry_issue1527_index_preserves_registration_order); + RUN_TEST(clsp_issue1527_cross_file_memo_failures_propagate); RUN_TEST(clsp_c_reassigned_function_pointer_calls_join_exact_occurrences); RUN_TEST(clsp_c_nested_function_pointer_shadow_restores_outer_target); RUN_TEST(clsp_cpp_repeated_same_leaf_calls_join_exact_occurrences); @@ -17264,4 +17739,6 @@ SUITE(c_lsp) { RUN_TEST(clsp_dll_multiple_functions); RUN_TEST(clsp_dll_func_ptr_typedef); RUN_TEST(clsp_easy_win_sfinaeconditional_return); + RUN_TEST(clsp_issue1527_fluent_chain_work_is_polynomial); + RUN_TEST(clsp_issue1527_scale_work_is_linear); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 4980426a92..cf35289210 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8919,6 +8919,51 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) { PASS(); } +/* #1527: the call-context metrics find each call's innermost enclosing + * Function/Method with a line sweep instead of scanning every def per call. + * The sweep must give the scan's exact answer -- smallest span, ties to the + * lowest def index -- on nested, overlapping, same-line, malformed + * (end < start) and unnamed/non-callable ranges. Fixed-seed generator: + * deterministic, many shapes. */ +static uint32_t enc_rand(uint32_t *state) { + *state = (*state * 1103515245u) + 12345u; + return (*state >> 16) & 0x7fffu; +} + +TEST(extract_enclosing_sweep_matches_scan_issue1527) { + static const char *const labels[] = {"Function", "Method", "Class", "Variable"}; + uint32_t seed = 1527; + for (int round = 0; round < 40; round++) { + CBMArena a; + cbm_arena_init(&a); + CBMFileResult r; + memset(&r, 0, sizeof(r)); + int ndefs = round == 0 ? 0 : (int)(enc_rand(&seed) % 300); + int ncalls = (int)(enc_rand(&seed) % 1500) + 1; + int max_line = (int)(enc_rand(&seed) % 400) + 5; + for (int i = 0; i < ndefs; i++) { + CBMDefinition d; + memset(&d, 0, sizeof(d)); + d.name = (enc_rand(&seed) % 10) == 0 ? NULL : "f"; + d.label = labels[enc_rand(&seed) % 4]; + d.start_line = 1 + (enc_rand(&seed) % (uint32_t)max_line); + int len = (int)(enc_rand(&seed) % 60) - 3; /* a few malformed end < start */ + d.end_line = (uint32_t)((int)d.start_line + len > 0 ? (int)d.start_line + len : 0); + cbm_defs_push(&r.defs, &a, d); + } + for (int i = 0; i < ncalls; i++) { + CBMCall c; + memset(&c, 0, sizeof(c)); + c.callee_name = "g"; + c.start_line = (int)(enc_rand(&seed) % (uint32_t)(max_line + 10)) - 2; + cbm_calls_push(&r.calls, &a, c); + } + ASSERT_EQ(cbm_test_enclosing_sweep_mismatches(&r), 0); + cbm_arena_destroy(&a); + } + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -9361,6 +9406,7 @@ SUITE(extraction) { RUN_TEST(json_toplevel_description_promoted_to_module_issue519); RUN_TEST(config_description_only_at_top_level_issue519); RUN_TEST(non_config_language_module_has_no_promoted_description_issue519); + RUN_TEST(extract_enclosing_sweep_matches_scan_issue1527); cbm_shutdown(); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9e26829e27..1fca364668 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10,7 +10,10 @@ #include "test_helpers.h" #include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test) #include "pipeline/pipeline.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "pipeline/lsp_surface.h" +#include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline_internal.h" #include "pipeline/artifact.h" #include "store/store.h" @@ -8166,6 +8169,168 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* Real cross-file memo failure, after successful extraction. The failure + * must reach the public per-file error list on both production drivers. */ +static int pipeline_lsp_failure_case(bool parallel, const char *stage, bool python, bool depth) { + const char *src = "struct S { int m() { return 1; } };\n" + "void run() { S s; s.m(); }\n"; + if (python) + src = "class S:\n def m(self):\n return 1\ndef run():\n s = S()\n s.m()\n"; + const char *filename = python ? "memo.py" : "memo.cpp"; + if (setup_usages_repo(filename, src, NULL, NULL) != 0) + FAIL("failed to create memo fixture"); + char path[512]; + if (parallel) { + for (int i = 0; i < 51; i++) { + snprintf(path, sizeof(path), "%s/memo_pad_%02d.%s", g_usages_tmpdir, i, + python ? "py" : "cpp"); + if (th_write_file(path, python ? "# inert parallel routing fixture\n" + : "// inert parallel routing fixture\n") != 0) { + teardown_usages_repo(); + FAIL("failed to create parallel routing fixture"); + } + } + } + const char *keys[] = { + "CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + depth ? (python ? "CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE" : "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE") + : (python ? "CBM_TEST_PY_LSP_MEMO_FAIL_STAGE" : "CBM_TEST_C_LSP_MEMO_FAIL_STAGE")}; + char *saved[3] = {0}; + bool env_saved = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + env_saved = env_saved && (!old || saved[i]); + } + if (!env_saved) { + for (int i = 0; i < 3; i++) + free(saved[i]); + teardown_usages_repo(); + FAIL("failed to save memo fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= cbm_setenv(keys[2], stage, 1); + snprintf(path, sizeof(path), "%s/memo.db", g_usages_tmpdir); + cbm_pipeline_t *p = set_rc == 0 ? cbm_pipeline_new(g_usages_tmpdir, path, CBM_MODE_FULL) : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(p, &errors, &error_count); + int memo_errors = 0; + const char *expected_phase = strcmp(stage, "cross") == 0 ? "lsp_skipped" : "extract"; + bool precise = true; + for (int i = 0; i < error_count; i++) { + if (errors[i].path && strcmp(errors[i].path, filename) == 0) { + memo_errors++; + precise = precise && errors[i].phase && strcmp(errors[i].phase, expected_phase) == 0 && + errors[i].reason && + strcmp(errors[i].reason, + depth ? (python ? CBM_PY_LSP_DEPTH_ERROR : CBM_C_LSP_DEPTH_ERROR) + : (python ? CBM_PY_LSP_MEMO_ERROR : CBM_C_LSP_MEMO_ERROR)) == 0; + } + } + cbm_pipeline_free(p); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + teardown_usages_repo(); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(memo_errors, 1); + ASSERT_TRUE(precise); + PASS(); +} + +TEST(pipeline_issue1527_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, false); +} +TEST(pipeline_issue1527_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, false); +} + +TEST(pipeline_issue1527_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, false); +} +TEST(pipeline_issue1527_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, false); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, true); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, true); +} + +TEST(pipeline_issue1527_cross_memo_failure_fallback) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_c_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_CPP, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + cbm_c_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16410,6 +16575,23 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { #endif SUITE(pipeline) { + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_fallback); RUN_TEST(pipeline_nested_fixture_files_are_written); RUN_TEST(pipeline_fixture_file_parent_is_preserved); RUN_TEST(pipeline_fixture_node_count_requires_exact_source_nodes); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index d65341900d..35dfe347b5 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -10,6 +10,7 @@ #include "test_framework.h" #include "cbm.h" #include "lsp/py_lsp.h" +#include "lsp/lsp_work.h" #include "pipeline/lsp_resolve.h" #include "pipeline/pass_lsp_cross.h" @@ -2223,16 +2224,30 @@ TEST(pylsp_issue710_heterogeneous_receiver_chain) { PASS(); } -TEST(pylsp_eval_steps_budget_degrades_gracefully) { - /* PY_EVAL_MAX_STEPS_PER_FILE guard: a file whose expressions demand - * more evaluator work than the per-file budget must still complete - * quickly and keep everything resolved BEFORE exhaustion — graceful - * partial degradation, not a hang, crash, or corrupted result. 90 - * statements x 30-link chains re-evaluated across the bind + emit - * phases (the assignment bind flushes the memo generation in between) - * comfortably exceed the 10000-step budget. */ - enum { BUDGET_STMTS = 90, BUDGET_LINKS = 30 }; - size_t sz = 4096 + (size_t)BUDGET_STMTS * (32 + (size_t)BUDGET_LINKS * 8); +/* Resolved INVOCATION rows whose callee QN ends with `suffix` (e.g. ".compile"). */ +static int count_resolved_suffix(const CBMFileResult *r, const char *suffix) { + size_t sl = strlen(suffix); + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + size_t ql = rc->callee_qn ? strlen(rc->callee_qn) : 0; + if (rc->kind == CBM_RESOLVED_INVOCATION && ql >= sl && + strcmp(rc->callee_qn + ql - sl, suffix) == 0) { + n++; + } + } + return n; +} + +TEST(pylsp_issue1527_chain_statements_resolve_completely) { + /* #1527 replaces the old budget-degradation guard: with no per-file + * evaluation budget, the #710 chain shape -- 90 statements x 30-link + * chains, re-evaluated across the bind + emit phases (the assignment bind + * flushes the memo generation in between) -- must still finish quickly + * AND resolve EVERY link and EVERY final call (each is a distinct call + * site), not merely the first statements. */ + enum { CHAIN_STMTS = 90, CHAIN_LINKS = 30 }; + size_t sz = 4096 + (size_t)CHAIN_STMTS * (32 + (size_t)CHAIN_LINKS * 8); char *src = malloc(sz); ASSERT_NOT_NULL(src); char *p = src; @@ -2247,11 +2262,11 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { "def run():\n"); p += n; left -= (size_t)n; - for (int s = 0; s < BUDGET_STMTS; s++) { + for (int s = 0; s < CHAIN_STMTS; s++) { n = snprintf(p, left, " v%d = G()", s); p += n; left -= (size_t)n; - for (int l = 0; l < BUDGET_LINKS; l++) { + for (int l = 0; l < CHAIN_LINKS; l++) { n = snprintf(p, left, ".add(%d)", l); p += n; left -= (size_t)n; @@ -2264,15 +2279,432 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { CBMFileResult *r = extract_py(src); free(src); ASSERT_NOT_NULL(r); - /* The first statements run with budget to spare — they must resolve. */ - ASSERT_GTE(require_resolved(r, "run", "G.compile"), 0); + ASSERT_EQ(count_resolved_suffix(r, ".G.compile"), CHAIN_STMTS); + ASSERT_EQ(count_resolved_suffix(r, ".G.add"), CHAIN_STMTS * CHAIN_LINKS); + cbm_free_result(r); + PASS(); +} + +TEST(pylsp_issue1527_large_module_resolves_every_call) { + /* #1527 reproduction: a module of 8000 classes plus one function calling + * `ClsN().chain().chain().method()` for each. Before the fix the per-file + * step cap stopped resolution after ~3333 statements (CALLS 14,666 of + * 24,000 in the graph) while the file still reported status `indexed`. + * Every one of the 8000 x 3 method call sites must resolve. */ + enum { N_CLASSES = 8000 }; + size_t sz = (size_t)N_CLASSES * 192 + 1024; + char *src = malloc(sz); + ASSERT_NOT_NULL(src); + size_t pos = (size_t)snprintf(src, sz, "from typing import Self\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + } + pos += (size_t)snprintf(src + pos, sz - pos, "def use():\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, " Cls%d().chain().chain().method()\n", i); + } + ASSERT(pos < sz); + CBMFileResult *r = + cbm_extract_file(src, (int)pos, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + free(src); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_resolved_suffix(r, ".method"), N_CLASSES); + ASSERT_EQ(count_resolved_suffix(r, ".chain"), 2 * N_CLASSES); + /* The last class is resolved exactly, not just some class. */ + ASSERT_GTE(require_resolved(r, "use", "Cls7999.method"), 0); cbm_free_result(r); PASS(); } /* ── Suite ─────────────────────────────────────────────────────── */ +static char *pylsp_work_fixture(int n_classes, int *out_len) { + /* Per class: ~140 chars (5-line def). Per call: ~50 chars. Overhead + * for the class number digits scales with log10(n) but the constant + * 256 covers up to 9-digit indices comfortably. */ + int approx = n_classes * 256 + 1024; + char *buf = (char *)malloc((size_t)approx); + if (!buf) + return NULL; + int pos = 0; + pos += snprintf(buf + pos, (size_t)(approx - pos), "from typing import Self\n"); + for (int i = 0; i < n_classes; i++) { + int n = snprintf(buf + pos, (size_t)(approx - pos), + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + if (n < 0 || pos + n >= approx) + break; + pos += n; + } + int n = snprintf(buf + pos, (size_t)(approx - pos), "def use():\n"); + pos += n; + for (int i = 0; i < n_classes; i++) { + int m = snprintf(buf + pos, (size_t)(approx - pos), + " Cls%d().chain().chain().method()\n", i); + if (m < 0 || pos + m >= approx) + break; + pos += m; + } + *out_len = pos; + return buf; +} + +/* Deterministic complexity gate (#1527, O9: no clock). The resolver + * primitives count every candidate they examine -- scope-frame probes, + * registry lookup probes, enclosing-definition candidates -- plus every real + * (non-memoized) expression evaluation, into a seam-build work counter. 4x + * the input must cost about 4x the work: a linear-scan lookup or a lost memo + * lands at ~16x (quadratic) or worse. Also asserts every call site resolved, + * so the work is not "linear" because resolution stopped early. */ +static uint64_t work_for(int n_classes, int *out_resolved) { + int slen = 0; + char *src = pylsp_work_fixture(n_classes, &slen); + if (!src) + return 0; + (void)cbm_lsp_work_take(); + CBMFileResult *r = + cbm_extract_file(src, slen, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + uint64_t work = cbm_lsp_work_take(); + *out_resolved = r ? r->resolved_calls.count : -1; + if (r) + cbm_free_result(r); + free(src); + return work; +} + +TEST(pylsp_scale_work_is_linear) { + int r1 = 0, r4 = 0; + uint64_t w1 = work_for(1000, &r1); + uint64_t w4 = work_for(4000, &r4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (resolved=%d) 4000=%llu (resolved=%d) ratio=%.2fx " + "(linear ~4x, quadratic ~16x)\n", + (unsigned long long)w1, r1, (unsigned long long)w4, r4, ratio); + /* Each class line: ClsN() + chain() + chain() + method() = 4 resolved sites. */ + ASSERT_EQ(r1, 4 * 1000); + ASSERT_EQ(r4, 4 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +extern const TSLanguage *tree_sitter_python(void); + +TEST(pylsp_issue1527_memo_allocation_failure_is_reported) { + int len = 0; + char *src = pylsp_work_fixture(200, &len); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int fail_after = 0; fail_after <= 1; fail_after++) { + cbm_py_lsp_test_memo_fail_after(fail_after); + CBMFileResult *r = extract_py(src); + cbm_py_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0; + if (r) { + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +TEST(pylsp_issue1527_memo_and_depth_failures_stop_work) { + const char *src = "x\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode node = ts_node_named_child(ts_node_named_child(ts_tree_root_node(tree), 0), 0); + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + const CBMType *known = cbm_type_named(&arena, "test.Known"); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.eval_depth = 256; + bool depth_unknown = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A fresh walk can resolve this shallow expression; the failed one cannot. */ + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + bool depth_recovered = cbm_py_lsp_test_eval(&ctx, node) == known && !ctx.eval_failure; + bool stopped = true; + for (int capacity_failure = 0; capacity_failure < 2; capacity_failure++) { + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.test_memo_allocations_left = 0; + /* Pin the maximum-capacity refusal without allocating a huge table. + * A NULL table bypasses lookup; insertion must report failed growth. */ + if (capacity_failure) + ctx.type_cache_cap = 1 << 26; + stopped = + stopped && cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + for (int i = 0; i < 20; i++) + stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && stopped; + stopped = stopped && cbm_lsp_work_take() == 0; + } + cbm_arena_destroy(&arena); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(depth_recovered); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class) { + const char *src = "class Local:\n pass\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse root-name fixture"); + } + bool closed = true, control = false; + for (int failure = 0; failure <= 2; failure++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.Local", .short_name = "Local"}); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.External", .short_name = "External"}); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + ctx.test_root_name_failure = failure; + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool local_absent = !cbm_scope_lookup_local(ctx.current_scope, "Local"); + bool external_present = cbm_scope_lookup_local(ctx.current_scope, "External") != NULL; + if (failure) + closed = + closed && ctx.callable_value_proof_disabled && local_absent && !external_present; + else + control = !ctx.callable_value_proof_disabled && local_absent && external_present; + cbm_arena_destroy(&arena); + } + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(control); + ASSERT_TRUE(closed); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows) { + char src[4096]; + size_t pos = (size_t)snprintf(src, sizeof(src), "def run(s: S):\n s + s\n"); + for (int i = 0; i < 200; i++) + pos += (size_t)snprintf(src + pos, sizeof(src) - pos, " s.m()\n"); + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.__add__", + .short_name = "__add__", + .label = "Method", + .receiver_type = "test.S", + .return_types = "S", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMResolvedCall prior = {.caller_qn = "test.run", + .callee_qn = "test.S.__add__", + .kind = CBM_RESOLVED_INVOCATION, + .confidence = 0.1f, + .strategy = "prior.strategy", + .reason = "prior.reason", + .site_start_byte = (uint32_t)(strstr(src, "s + s") - src), + .site_end_byte = (uint32_t)(strstr(src, "s + s") - src) + 5}; + CBMCall prior_synthetic = {.callee_name = "completed.synthetic"}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + CBMCallArray synthetic = {.items = &prior_synthetic, .count = 1, .cap = 1}; + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 3) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 3}; + cbm_py_lsp_test_memo_fail_after(1); + bool complete = + mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, NULL, + NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test", + &overlay, NULL, NULL, 0, NULL, &out, + &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + cbm_py_lsp_test_memo_fail_after(-1); + correct = correct && !complete && out.count == (mode == 2 ? 0 : 1) && + synthetic.count == 1 && prior.confidence == 0.1f && + strcmp(prior.strategy, "prior.strategy") == 0 && + strcmp(prior.reason, "prior.reason") == 0 && + strcmp(prior_synthetic.callee_name, "completed.synthetic") == 0; + if (mode != 2) + correct = correct && out.items[0].confidence == 0.1f && + strcmp(out.items[0].strategy, "prior.strategy") == 0 && + strcmp(out.items[0].reason, "prior.reason") == 0; + if (mode == 2) + out = (CBMResolvedCallArray){0}; + complete = mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, + NULL, NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + correct = correct && complete && out.count > 200 && + (mode == 2 || (synthetic.count > 1 && out.items[0].confidence > 0.1f)); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error) { + const char *src = "def run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_py_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_PYTHON, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + cbm_py_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(pylsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "class S:\n def m(self):\n return 1\ndef run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + cbm_py_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_py(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2}; + if (mode == 2) { + correct = !cbm_batch_py_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_py_lsp_cross_status(&arena, src, (int)strlen(src), "test", defs, + 2, NULL, NULL, 0, NULL, &out, NULL) + : cbm_run_py_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, + 0, NULL, &out, NULL); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_PYTHON, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_py_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error); + RUN_TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows); + RUN_TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class); + RUN_TEST(pylsp_issue1527_memo_and_depth_failures_stop_work); + RUN_TEST(pylsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(pylsp_scale_work_is_linear); /* Phase 2 — smoke */ RUN_TEST(pylsp_smoke_empty); RUN_TEST(pylsp_smoke_one_function); @@ -2366,5 +2798,6 @@ SUITE(py_lsp) { /* Evaluator guards — #710/#720 (distilled PRs #732 + #758) */ RUN_TEST(pylsp_issue710_deep_call_chain_resolves); RUN_TEST(pylsp_issue710_heterogeneous_receiver_chain); - RUN_TEST(pylsp_eval_steps_budget_degrades_gracefully); + RUN_TEST(pylsp_issue1527_chain_statements_resolve_completely); + RUN_TEST(pylsp_issue1527_large_module_resolves_every_call); } diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.c index f20e762753..1982e0623a 100644 --- a/tests/test_py_lsp_scale.c +++ b/tests/test_py_lsp_scale.c @@ -1,7 +1,7 @@ /* * test_py_lsp_scale.c — measure scaling behavior at 100 / 500 / 2000 - * classes-and-calls. Asserts that doubling the input doesn't more than - * 4x the runtime (catches accidental O(n^2) in the resolver). + * classes-and-calls. Timing is diagnostic; deterministic work is gated in + * the normal py_lsp suite. */ #include "test_framework.h" #include "cbm.h" @@ -88,26 +88,10 @@ TEST(pylsp_scale_linear_growth) { ASSERT(r_pct_100 > 0.5); ASSERT(r_pct_2000 > 0.5); - /* Quadratic-growth detector. 20x input: linear ~20-30x time, clear - * quadratic ~400x. The bound sits at 200x — mid-way in log space — so a - * real quadratic regression still fails by 2x while the CURRENT, KNOWN - * superlinear resolve curve does not produce false release blocks. - * - * The old bound of 100x was calibrated as "linear plus generous overhead", - * but the resolve path has never been linear here: measured 2026-08-11 - * (sanitized builds, deterministic across runs) — quiet arm64 host - * 57-68x, macos-15-intel CI runner 101.1x, per-function cost growing - * 0.5ms -> 1.7ms from 100 to 2000 functions. That ~O(n^1.4-1.5) curve is - * the audited short-name-lookup/negative-memo gap, tracked as #1527; this - * detector was flagging host CONSTANTS, not a complexity change. When - * #1527 lands, tighten this back down (~40x holds linear honestly). */ - if (t100 > 0.5) { // skip when t100 too small to compare reliably - double ratio = t2000 / t100; - printf(" scale ratio 2000/100: %.1fx (linear ~20x, known-superlinear ~60-100x, " - "quadratic ~400x)\n", - ratio); - ASSERT(ratio < 200.0); // flags clear quadratic; #1527 tracks the curve itself - } + /* Timing is diagnostic only. The normal py_lsp suite gates deterministic + * resolver work and complete resolution independently of host scheduling. */ + if (t100 > 0.0) + printf(" diagnostic scale ratio 2000/100: %.1fx\n", t2000 / t100); PASS(); } From 90132b5fe401d50bc5d2f3aceaabca5a3267bbae Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 20:38:20 +0200 Subject: [PATCH 4/9] fix(python): join module-level calls to exact LSP resolutions Preserve the arena-owned raw Module caller name so top-level resolutions join the extractor in both pipeline drivers. Add occurrence and ownership controls, including package modules and named lambdas. Signed-off-by: Martin Vogel --- internal/cbm/lsp/py_lsp.c | 4 +- tests/test_pipeline.c | 201 ++++++++++++++++++++++++++++++++++ tests/test_py_lsp.c | 219 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+), 1 deletion(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index e78bba7dcb..48cadcd45f 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -4817,7 +4817,9 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { uint32_t nc = rk.n; const char *prev_func = ctx->enclosing_func_qn; - ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.__module__", ctx->module_qn); + /* Extraction uses the file's Module QN at top level. Keep the caller + * arena-owned for cross-file APIs whose module_qn input may be temporary. */ + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 1fca364668..ba220bc84b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8331,6 +8331,205 @@ TEST(pipeline_issue1527_cross_memo_failure_fallback) { PASS(); } +static bool python_module_write_fixture_file(const char *root, const char *name, const char *text) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/%s", root, name); + return n > 0 && (size_t)n < sizeof(path) && th_write_file(path, text) == 0; +} + +/* An absent node, failed edge query, or heuristic edge must not satisfy this + * integration check. Exact source/target identities precede strategy checks. */ +static bool python_module_exact_lsp_edge(cbm_store_t *store, const char *project, + const char *caller, const char *caller_label, + const char *target, const char *target_label) { + char source_qn[768], target_qn[768]; + int sn = snprintf(source_qn, sizeof(source_qn), "%s.%s", project, caller); + int tn = snprintf(target_qn, sizeof(target_qn), "%s.%s", project, target); + if (sn <= 0 || (size_t)sn >= sizeof(source_qn) || tn <= 0 || (size_t)tn >= sizeof(target_qn)) + return false; + cbm_node_t source = {0}, destination = {0}; + bool correct = + cbm_store_find_node_by_qn(store, project, source_qn, &source) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, target_qn, &destination) == CBM_STORE_OK && + source.id > 0 && destination.id > 0 && source.label && destination.label && + strcmp(source.label, caller_label) == 0 && strcmp(destination.label, target_label) == 0; + cbm_edge_t *edges = NULL; + int edge_count = 0, hits = 0; + if (correct) { + correct = cbm_store_find_edges_by_source_type(store, source.id, "CALLS", &edges, + &edge_count) == CBM_STORE_OK; + for (int i = 0; correct && i < edge_count; i++) { + if (edges[i].source_id != source.id || edges[i].target_id != destination.id) + continue; + hits++; + const char *properties = edges[i].properties_json; + yyjson_doc *doc = properties ? yyjson_read(properties, strlen(properties), 0) : NULL; + const char *strategy = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "strategy")) : NULL; + correct = strategy && strncmp(strategy, "lsp_", 4) == 0; + if (doc) + yyjson_doc_free(doc); + } + } + cbm_store_free_edges(edges, edge_count); + cbm_node_free_fields(&source); + cbm_node_free_fields(&destination); + return correct && hits == 1; +} + +static int python_module_callers_pipeline_case(bool parallel) { + static const char helpers[] = "def helper():\n" + " return 1\n" + "class Service:\n" + " def run(self):\n" + " return 2\n"; + static const char app[] = "from helpers import helper, Service\n" + "def local_helper():\n" + " return 3\n" + "class Local:\n" + " def run(self):\n" + " return 4\n" + "class Controller:\n" + " def run(self):\n" + " helper()\n" + " method_service = Service()\n" + " method_service.run()\n" + "def control():\n" + " helper()\n" + " function_service = Service()\n" + " function_service.run()\n" + " local_helper()\n" + " function_local = Local()\n" + " function_local.run()\n" + "helper()\n" + "module_service = Service()\n" + "module_service.run()\n" + "local_helper()\n" + "module_local = Local()\n" + "module_local.run()\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } expected[] = { + {"app", "Module", "helpers.helper", "Function"}, + {"app", "Module", "helpers.Service.run", "Method"}, + {"app", "Module", "app.local_helper", "Function"}, + {"app", "Module", "app.Local.run", "Method"}, + {"app.control", "Function", "helpers.helper", "Function"}, + {"app.control", "Function", "helpers.Service.run", "Method"}, + {"app.control", "Function", "app.local_helper", "Function"}, + {"app.control", "Function", "app.Local.run", "Method"}, + {"app.Controller.run", "Method", "helpers.helper", "Function"}, + {"app.Controller.run", "Method", "helpers.Service.run", "Method"}, + {"pkg.__init__", "Module", "helpers.helper", "Function"}, + }; + char root[512]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-module-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python module fixture"); + char path[768]; + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file(root, "helpers.py", helpers) && + python_module_write_fixture_file(root, "app.py", app) && + python_module_write_fixture_file(root, "pkg/__init__.py", + "from helpers import helper\n" + "PACKAGE_VALUE = helper()\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "module_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "padding = 0\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python module fixture"); + } + + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {0}; + bool saved_ok = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + free(saved[0]); + free(saved[1]); + th_rmtree(root); + FAIL("failed to save Python module fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + n = snprintf(path, sizeof(path), "%s/module.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + int file_count = 0; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (parallel ? 55 : 3); + cbm_store_free_nodes(files, file_count); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + bool edge = project && python_module_exact_lsp_edge( + store, project, expected[i].caller, expected[i].caller_label, + expected[i].target, expected[i].target_label); + correct = correct && edge; + } + /* Neither the ordinary nor package Module should acquire a fake + * __module__ owner node. A failed lookup query is not absence proof. */ + const char *phantoms[] = {"app.__module__", "pkg.__init__.__module__"}; + for (int i = 0; project && i < 2; i++) { + char qn[768]; + n = snprintf(qn, sizeof(qn), "%s.%s", project, phantoms[i]); + cbm_node_t node = {0}; + bool absent = + n > 0 && (size_t)n < sizeof(qn) && + cbm_store_find_node_by_qn(store, project, qn, &node) == CBM_STORE_NOT_FOUND; + correct = correct && absent; + cbm_node_free_fields(&node); + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, 0); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pipeline_python_module_callers_sequential) { + return python_module_callers_pipeline_case(false); +} + +TEST(pipeline_python_module_callers_parallel) { + return python_module_callers_pipeline_case(true); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16575,6 +16774,8 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { #endif SUITE(pipeline) { + RUN_TEST(pipeline_python_module_callers_sequential); + RUN_TEST(pipeline_python_module_callers_parallel); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 35dfe347b5..adc84913b3 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -2697,7 +2697,226 @@ TEST(pylsp_issue1527_depth_failure_reports_all_entry_points) { PASS(); } +/* Join one known source occurrence, not merely two equal aggregate counts. + * Changing the owner or range must not borrow a neighbouring resolution. */ +static bool pylsp_module_site_joins(const CBMCallArray *calls, const CBMResolvedCallArray *resolved, + const char *source, const char *site_text, const char *caller, + const char *target) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source); + uint32_t end = start + (uint32_t)strlen(site_text); + const CBMCall *call = NULL; + int call_count = 0; + for (int i = 0; i < calls->count; i++) { + const CBMCall *candidate = &calls->items[i]; + if (candidate->enclosing_func_qn && strcmp(candidate->enclosing_func_qn, caller) == 0 && + candidate->site_start_byte == start && candidate->site_end_byte == end) { + call = candidate; + call_count++; + } + } + if (call_count != 1) + return false; + const CBMResolvedCall *hit = cbm_pipeline_find_lsp_resolution(resolved, call, false); + if (!hit || !hit->caller_qn || strcmp(hit->caller_qn, caller) != 0 || !hit->callee_qn || + strcmp(hit->callee_qn, target) != 0 || hit->kind != CBM_RESOLVED_INVOCATION || + hit->site_start_byte != start || hit->site_end_byte != end || + hit->source_origin != call->source_origin) + return false; + int semantic_count = 0; + for (int i = 0; i < resolved->count; i++) { + const CBMResolvedCall *candidate = &resolved->items[i]; + if (candidate->kind == CBM_RESOLVED_INVOCATION && candidate->caller_qn && + strcmp(candidate->caller_qn, caller) == 0 && candidate->site_start_byte == start && + candidate->site_end_byte == end) + semantic_count++; + } + CBMCall wrong = *call; + wrong.enclosing_func_qn = "unrelated.module.caller"; + bool rejects_owner = cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; + wrong = *call; + wrong.site_start_byte++; + return semantic_count == 1 && rejects_owner && + cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; +} + +static bool pylsp_module_has_def(const CBMFileResult *r, const char *qn) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].qualified_name && strcmp(r->defs.items[i].qualified_name, qn) == 0) + return true; + } + return false; +} + +TEST(pylsp_module_callers_join_exact_raw_occurrences) { + static const char source[] = "class Alpha:\n" + " def render(self):\n" + " return 1\n" + "class Beta:\n" + " def render(self):\n" + " return 2\n" + "def helper():\n" + " return 3\n" + "def control():\n" + " function_a = Alpha()\n" + " function_a.render()\n" + " helper()\n" + "class Controller:\n" + " def control(self):\n" + " method_b = Beta()\n" + " method_b.render()\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.app") == 0; + if (correct) { + correct = + pylsp_module_has_def(r, "test.app.Alpha.render") && + pylsp_module_has_def(r, "test.app.Beta.render") && + pylsp_module_has_def(r, "test.app.control") && + pylsp_module_has_def(r, "test.app.Controller.control") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_a.render()", + "test.app", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_b.render()", + "test.app", "test.app.Beta.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "function_a.render()", + "test.app.control", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "method_b.render()", + "test.app.Controller.control", "test.app.Beta.render"); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + + /* Package symbol-scope normalization is separate. A builtin isolates + * this package Module's raw caller identity from that remaining issue. */ + static const char init_source[] = "PACKAGE_SIZE = len([1])\n"; + r = cbm_extract_file(init_source, (int)strlen(init_source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.pkg.__init__") == 0 && + pylsp_module_has_def(r, "builtins.len") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, init_source, "len([1])", + "test.pkg.__init__", "builtins.len"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair) { + static const char source[] = "class Number:\n" + " def __add__(self, other):\n" + " return self\n" + "module_number = Number()\n" + "add = lambda left, right: left + right\n" + "first = add(module_number, module_number)\n" + "second = add(module_number, module_number)\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + pylsp_module_has_def(r, "test.app.Number.__add__") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "left + right", + "test.app.", "test.app.Number.__add__"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites) { + static const char source[] = "from helpers import Alpha, Beta\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + static const char *const paths[] = {"app.py", "pkg/__init__.py"}; + static const char *const modules[] = {"test.app", "test.pkg.__init__"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.helpers.Alpha", + .short_name = "Alpha", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Alpha.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Alpha", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta", + .short_name = "Beta", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Beta", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *import_names[] = {"Alpha", "Beta"}; + const char *import_qns[] = {"test.helpers.Alpha", "test.helpers.Beta"}; + bool correct = true; + for (int path = 0; path < 2; path++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[path], 0, NULL, NULL); + if (!r || r->has_error || r->parse_incomplete || r->lsp_skipped || !r->module_qn || + strcmp(r->module_qn, modules[path]) != 0) { + correct = false; + if (r) + cbm_free_result(r); + continue; + } + for (int prebuilt = 0; prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = prebuilt ? cbm_py_build_cross_registry(&shared, defs, 4) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + char module_input[64]; + snprintf(module_input, sizeof(module_input), "%s", modules[path]); + CBMResolvedCallArray out = {0}; + CBMLSPStatus status = + prebuilt ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), module_input, &overlay, + import_names, import_qns, 2, NULL, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + module_input, defs, 4, import_names, + import_qns, 2, NULL, &out, NULL); + /* Returned caller strings must outlive the caller's input buffer. + * Overwrite live storage, avoiding a dangling-pointer test. */ + memset(module_input, 'x', strlen(module_input)); + correct = correct && (!prebuilt || (base && base->read_only)) && + status == CBM_LSP_COMPLETE && + pylsp_module_site_joins(&r->calls, &out, source, "module_a.render()", + modules[path], "test.helpers.Alpha.render") && + pylsp_module_site_joins(&r->calls, &out, source, "module_b.render()", + modules[path], "test.helpers.Beta.render"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_module_callers_join_exact_raw_occurrences); + RUN_TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair); + RUN_TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites); RUN_TEST(pylsp_issue1527_depth_failure_reports_all_entry_points); RUN_TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error); RUN_TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows); From af6af273475e046565d1c3e3d90a8f00bce199fb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 21:14:20 +0200 Subject: [PATCH 5/9] fix(python): separate package symbol scope from file identity Resolve symbols declared in __init__.py under their package while retaining the raw module identity for file and top-level caller nodes. Carry explicit language through registry callers, select Python package imports, preserve long exact names, and report scope allocation failures without partial output. Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 6 + internal/cbm/helpers.c | 11 + internal/cbm/lsp/py_lsp.c | 126 +++++++--- internal/cbm/lsp/py_lsp.h | 9 +- src/pipeline/pass_calls.c | 25 +- src/pipeline/pass_lsp_cross.c | 77 ++++-- src/pipeline/pass_parallel.c | 71 +++--- src/pipeline/pass_semantic.c | 26 +- src/pipeline/pass_usages.c | 18 +- src/pipeline/pipeline_internal.h | 9 + src/pipeline/registry.c | 105 +++++--- tests/test_pipeline.c | 356 +++++++++++++++++++++++++++ tests/test_py_lsp.c | 396 +++++++++++++++++++++++++++++++ tests/test_registry.c | 270 +++++++++++++++++++++ 14 files changed, 1367 insertions(+), 138 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6cf6d04fec..56f57dcd4a 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -456,6 +456,7 @@ typedef enum { CBM_LSP_COMPLETE = 0, CBM_LSP_MEMO_FAILED, CBM_LSP_DEPTH_EXCEEDED, + CBM_LSP_SCOPE_FAILED, } CBMLSPStatus; // Growable arrays used during extraction. @@ -982,4 +983,9 @@ bool cbm_label_is_relation(const char *label); // `label` may be NULL (returns false). Defined in helpers.c. bool cbm_label_is_registry_symbol(const char *label); +/* Python symbol scope only: callers must establish the language explicitly. + * A trailing .__init__ is omitted when a nonempty prefix precedes it; raw + * Module/file identity is unchanged. NULL returns zero. */ +size_t cbm_fqn_symbol_scope_len(const char *module_qn); + #endif // CBM_H diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 4c4d340f0b..4ab8ba48e7 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1611,6 +1611,17 @@ char *cbm_fqn_compute(CBMArena *a, const char *project, const char *rel_path, co return buf; } +size_t cbm_fqn_symbol_scope_len(const char *module_qn) { + if (!module_qn) + return 0; + size_t len = strlen(module_qn); + static const char suffix[] = ".__init__"; + size_t suffix_len = sizeof(suffix) - 1; + if (len > suffix_len && memcmp(module_qn + len - suffix_len, suffix, suffix_len) == 0) + return len - suffix_len; + return len; +} + char *cbm_fqn_module(CBMArena *a, const char *project, const char *rel_path) { return cbm_fqn_compute(a, project, rel_path, NULL); } diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 48cadcd45f..29cec01e73 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -39,6 +39,10 @@ #ifdef CBM_ENABLE_TEST_SEAMS static _Thread_local int py_test_memo_allocations_left = -1; static _Thread_local bool py_test_depth_failure; +static _Thread_local int py_test_scope_allocations_left = -1; +void cbm_py_lsp_test_scope_fail_after(int successful_allocations) { + py_test_scope_allocations_left = successful_allocations; +} void cbm_py_lsp_test_depth_fail(bool enabled) { py_test_depth_failure = enabled; } @@ -65,8 +69,9 @@ void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { result->has_error = true; result->lsp_skipped = true; /* Static lifetime also works when the result arena cannot allocate. */ - result->error_msg = - status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR : CBM_PY_LSP_MEMO_ERROR; + result->error_msg = status == CBM_LSP_SCOPE_FAILED ? CBM_PY_LSP_SCOPE_ERROR + : status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR + : CBM_PY_LSP_MEMO_ERROR; } // Forward decls @@ -290,9 +295,34 @@ static void py_scope_restore(PyLSPContext *ctx, CBMScope *saved) { ctx->current_scope = saved; } -void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, - const CBMTypeRegistry *registry, const char *module_qn, - CBMResolvedCallArray *out) { +/* Only a changed Python scope needs storage. Failure is explicit: never + * register or walk a package using the raw file scope as an OOM fallback. */ +static bool py_symbol_scope(CBMArena *arena, const char *module_qn, const char *stage, + const char **out) { + *out = module_qn; + size_t len = cbm_fqn_symbol_scope_len(module_qn); + if (!module_qn || module_qn[len] == '\0') + return true; +#ifdef CBM_ENABLE_TEST_SEAMS + const char *fail_stage = getenv("CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"); + if ((fail_stage && strcmp(fail_stage, stage) == 0) || py_test_scope_allocations_left == 0) + return false; + if (py_test_scope_allocations_left > 0) + py_test_scope_allocations_left--; +#else + (void)stage; +#endif + char *scope = cbm_arena_strndup(arena, module_qn, len); + if (!scope) + return false; + *out = scope; + return true; +} + +static void py_lsp_init_for_stage(PyLSPContext *ctx, CBMArena *arena, const char *source, + int source_len, const CBMTypeRegistry *registry, + const char *module_qn, CBMResolvedCallArray *out, + const char *stage) { if (!ctx) return; memset(ctx, 0, sizeof(PyLSPContext)); @@ -306,13 +336,23 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou ctx->source_len = source_len; ctx->registry = registry; ctx->registry_head = (CBMTypeRegistry *)registry; - ctx->module_qn = module_qn; + ctx->file_module_qn = module_qn; ctx->resolved_calls = out; + if (!py_symbol_scope(arena, module_qn, stage, &ctx->module_qn)) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return; + } ctx->current_scope = py_scope_push_checked(ctx); const char *dbg = getenv("CBM_LSP_DEBUG"); ctx->debug = dbg && dbg[0] && dbg[0] != '0'; } +void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, + const CBMTypeRegistry *registry, const char *module_qn, + CBMResolvedCallArray *out) { + py_lsp_init_for_stage(ctx, arena, source, source_len, registry, module_qn, out, "context"); +} + void py_lsp_add_import(PyLSPContext *ctx, const char *local_name, const char *module_qn) { if (!ctx || !local_name || !module_qn) return; @@ -2836,17 +2876,18 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // segment) and look up ".". A genuinely-external module // (requests, os) has no such project def, so it correctly stays // lsp_module_attr_unresolved. - if (mod && ctx->module_qn) { - const char *last_dot = strrchr(ctx->module_qn, '.'); - if (last_dot && last_dot > ctx->module_qn) { - size_t root_len = (size_t)(last_dot - ctx->module_qn); + if (mod && ctx->file_module_qn) { + const char *last_dot = strrchr(ctx->file_module_qn, '.'); + if (last_dot && last_dot > ctx->file_module_qn) { + size_t root_len = (size_t)(last_dot - ctx->file_module_qn); // Skip if mod is already rooted under the project to avoid // "..mod". - if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) { + if (!(strncmp(mod, ctx->file_module_qn, root_len) == 0 && + mod[root_len] == '.')) { char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); if (qual_mod) { - memcpy(qual_mod, ctx->module_qn, root_len); + memcpy(qual_mod, ctx->file_module_qn, root_len); qual_mod[root_len] = '.'; strcpy(qual_mod + root_len + 1, mod); const CBMRegisteredFunc *qf = @@ -4305,6 +4346,7 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) return; } size_t prefix_len = strlen(ctx->module_qn); + bool package_scope = ctx->file_module_qn && strcmp(ctx->file_module_qn, ctx->module_qn) != 0; CBMTypeShortIter all_types; cbm_registry_all_types_chain(ctx->registry, &all_types); for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { @@ -4315,6 +4357,8 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) cbm_scope_lookup_local(defined, name)) { continue; } + if (package_scope && strchr(qn + prefix_len + 1, '.')) + continue; /* submodule/nested classes are not package globals */ py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); } } @@ -4819,7 +4863,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { const char *prev_func = ctx->enclosing_func_qn; /* Extraction uses the file's Module QN at top level. Keep the caller * arena-owned for cross-file APIs whose module_qn input may be temporary. */ - ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->module_qn); + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->file_module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { @@ -5053,6 +5097,16 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, if (!arena || !result || result->lsp_skipped) return; + CBMTypeRegistry reg; + cbm_registry_init(®, arena); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, result->module_qn, + &result->resolved_calls, "raw"); + if (ctx.eval_failure) { + cbm_py_lsp_record_failure(result, ctx.eval_failure); + return; + } + /* Inject minimal builtin definitions as real graph nodes (builtins.len, * builtins.str, builtins.str.upper, ...). The typeshed registry already * RESOLVES builtin calls (emitting the strategy + a "builtins.*" callee_qn), @@ -5062,12 +5116,9 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, * builtin/constructor/method edges point at. Upsert dedups by QN. */ py_builtins_inject_defs(result, arena); - CBMTypeRegistry reg; - cbm_registry_init(®, arena); - cbm_python_stdlib_register(®, arena); - const char *module_qn = result->module_qn; + const char *module_qn = ctx.module_qn; // Register the file's own definitions so calls inside this file can // resolve via the registry. @@ -5088,8 +5139,6 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, reg.index_first_registered = true; cbm_registry_finalize_into(®, &idx_arena); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, &result->resolved_calls); /* Let the resolver inject synthetic syntactic calls for operator/subscript * dunder desugaring so those recovered calls reach the CALLS-edge pipeline. */ ctx.syn_calls = &result->calls; @@ -5187,7 +5236,7 @@ static void py_split_field_defs(CBMArena *arena, const char *field_defs, const c /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ -static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, +static bool py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, CBMLSPDef *defs, int def_count) { /* Pass 1: types only — the method pass probes the registry per Method def * (receiver auto-registration), which is a LINEAR scan pre-finalize: @@ -5251,7 +5300,13 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe ret_types[n] = NULL; } } - PySignatureParamParserContext parser_ctx = {.module_qn = d->def_module_qn}; + const char *symbol_scope = d->def_module_qn; + /* Fallback definitions can be mixed-language. Only authoritative + * Python metadata opts into package scope normalization. */ + if (d->lang == CBM_LANG_PYTHON && + !py_symbol_scope(arena, d->def_module_qn, "register", &symbol_scope)) + return false; + PySignatureParamParserContext parser_ctx = {.module_qn = symbol_scope}; const CBMType **param_types = cbm_type_materialize_signature_params( arena, d->signature_param_types, d->signature_param_count, py_signature_param_type_adapter, &parser_ctx); @@ -5271,8 +5326,16 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe cbm_registry_add_func(reg, rf); } } + return true; } +#ifdef CBM_ENABLE_TEST_SEAMS +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count) { + return py_register_lsp_defs(arena, NULL, reg, defs, def_count); +} +#endif + bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, @@ -5315,15 +5378,20 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in /* Index allocations go to a per-call scratch arena (see php_lsp_cross). */ CBMArena idx_arena; cbm_arena_init(&idx_arena); - py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, module_qn, out, "cross"); + if (ctx.eval_failure) + goto cross_cleanup; + if (!py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count)) { + ctx.eval_failure = CBM_LSP_SCOPE_FAILED; + goto cross_cleanup; + } py_mark_ambiguous_callable_bindings(®); /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" * comment for the rationale. */ cbm_registry_finalize_into(®, &idx_arena); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, out); ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { @@ -5332,6 +5400,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in } py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); +cross_cleanup: cbm_arena_destroy(&idx_arena); if (owns_tree && tree) @@ -5357,7 +5426,8 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i if (d->lang != CBM_LANG_PYTHON) continue; /* Reuse the existing register fn on a single-def slice (n=1 inline). */ - py_register_lsp_defs(arena, NULL, reg, d, 1); + if (!py_register_lsp_defs(arena, NULL, reg, d, 1)) + return NULL; /* caller owns the shared arena and other registries */ } py_mark_ambiguous_callable_bindings(reg); @@ -5401,7 +5471,9 @@ CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( TSNode root = ts_tree_root_node(tree); PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, reg, module_qn, out); + py_lsp_init_for_stage(&ctx, arena, source, source_len, reg, module_qn, out, "cross"); + if (ctx.eval_failure) + goto prebuilt_cleanup; ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { @@ -5410,7 +5482,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( } py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); - +prebuilt_cleanup: if (owns_tree && tree) ts_tree_delete(tree); if (parser) diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index e7536b8f9e..d733b31538 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -61,7 +61,8 @@ typedef struct { // caller QNs. const char *enclosing_func_qn; const char *enclosing_class_qn; - const char *module_qn; + const char *module_qn; // Python symbol scope (package __init__ omitted) + const char *file_module_qn; // raw file identity; borrowed for this API call // Output: resolved calls accumulate here. CBMResolvedCallArray *resolved_calls; @@ -102,7 +103,7 @@ typedef struct { uint32_t type_cache_gen; // bumped on every scope mutation (O(1) flush) int eval_depth; // evaluator recursion depth (PY_LSP_MAX_EVAL_DEPTH) uint32_t eval_truncations; // depth-cap cutoff count — gates memo inserts - CBMLSPStatus eval_failure; // sticky: memo/depth failure makes this walk incomplete + CBMLSPStatus eval_failure; // sticky: explicit failure makes this walk incomplete #ifdef CBM_ENABLE_TEST_SEAMS int test_memo_allocations_left; int test_root_name_failure; // 1 = NULL, 2 = empty; only the root-name collection @@ -167,10 +168,14 @@ void cbm_python_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); #define CBM_PY_LSP_DEPTH_ERROR "Python expression recursion depth limit exceeded" #define CBM_PY_LSP_MEMO_ERROR "Python expression memo allocation or capacity failure" +#define CBM_PY_LSP_SCOPE_ERROR "Python symbol scope allocation failure" void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); #ifdef CBM_ENABLE_TEST_SEAMS void cbm_py_lsp_test_memo_fail_after(int successful_allocations); void cbm_py_lsp_test_depth_fail(bool enabled); +void cbm_py_lsp_test_scope_fail_after(int successful_allocations); +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count); const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root); #endif diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f068ad4112..1c0dc1a62e 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -377,7 +377,8 @@ bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *proje /* Handle a route registration call: create Route node + HANDLES edge. */ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char route_qn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -397,8 +398,8 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca esc_fa); cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, - module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t hres = cbm_registry_resolve_lang( + ctx->registry, call->second_arg_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *handler = cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); if (handler != NULL) { @@ -588,10 +589,11 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count); + handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count, + lang); return; } if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { @@ -675,7 +677,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM res.strategy = lsp->strategy; res.candidate_count = 1; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, - imp_vals, imp_count, false); + imp_vals, imp_count, false, lang); return SKIP_ONE; } } @@ -741,8 +743,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM lang, lsp->strategy, lsp->callee_qn, ctx->project_name); cbm_resolution_t res = {0}; if (!rust_external) { - res = cbm_registry_resolve(ctx->registry, call->callee_name, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } if (!res.qualified_name || res.qualified_name[0] == '\0') { /* Resolution is empty when the callee belongs to an EXTERNAL client @@ -767,7 +769,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && call->first_string_arg[0] == '/') { handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count); + imp_count, lang); return SKIP_ONE; } cbm_svc_kind_t esvc = cbm_service_pattern_match(call->callee_name); @@ -888,7 +890,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, drop_plain_call); + imp_count, drop_plain_call, lang); return SKIP_ONE; } @@ -1125,7 +1127,8 @@ static int scan_depends_in_sig(cbm_pipeline_ctx_t *ctx, const cbm_regex_t *re, c } memcpy(func_ref, scan + match[SKIP_ONE].rm_so, (size_t)ref_len); func_ref[ref_len] = '\0'; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, func_ref, module_qn, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_ref, module_qn, ik, iv, + ic, CBM_LANG_PYTHON); if (res.qualified_name && res.qualified_name[0] != '\0') { const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_qn(ctx->gbuf, def->qualified_name); const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name); diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 6629956e7e..643d3ff191 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -227,11 +227,12 @@ static bool pxc_base_strategy_is_weak(const char *strategy) { * stdlib and third-party bases land here and keep their raw spelling. */ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { if (!reg || !raw || !raw[0]) { return NULL; } - cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = + cbm_registry_resolve_lang(reg, raw, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || !res.qualified_name[0]) { return NULL; } @@ -249,7 +250,8 @@ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *ra * registry does not know keeps working exactly as before. */ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, const cbm_registry_t *reg, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { if (!bases || !bases[0]) { return NULL; } @@ -264,7 +266,7 @@ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, } for (int i = 0; i < count; i++) { const char *qn = - pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count, lang); resolved[i] = qn ? qn : bases[i]; } resolved[count] = NULL; @@ -419,7 +421,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch * raw source spelling their own registrar already knows how to handle. */ dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, - imp_keys, imp_vals, imp_count) + imp_keys, imp_vals, imp_count, lang) : pxc_join_pipe(arena, src->base_classes); dst->signature_param_types = NULL; dst->signature_param_count = 0; @@ -622,8 +624,9 @@ static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result fields[f].type_qn = NULL; CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; - const char *qn = - name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + const char *qn = name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, + imp_count, CBM_LANG_PYTHON) + : NULL; if (!qn || !ft->field_name || !ft->field_name[0]) { continue; } @@ -2037,16 +2040,33 @@ static void pxc_mark_module_defs(const CBMModuleDefIndex *idx, bool *selected, * (`project.module`). Select the nearest materialized module prefix; the * language registry still has to prove the full target QN before an edge is * emitted, so this broadens the candidate set without weakening precision. */ -static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int pxc_import_probe_allocations_left = -1; +void cbm_pxc_test_import_probe_fail_after(int successful_allocations) { + pxc_import_probe_allocations_left = successful_allocations; +} +#endif + +static bool pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, const CBMLSPDef *all_defs, CBMLanguage caller_lang, const char *import_qn, int *total) { if (!idx || !idx->ht || !import_qn || !import_qn[0]) { - return; - } - char *candidate = strdup(import_qn); - if (!candidate) { - return; + return true; } + static const char init_seg[] = ".__init__"; + size_t import_len = strlen(import_qn); + if (import_len > SIZE_MAX - sizeof(init_seg)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (pxc_import_probe_allocations_left == 0) + return false; + if (pxc_import_probe_allocations_left > 0) + pxc_import_probe_allocations_left--; +#endif + char *candidate = (char *)malloc(import_len + sizeof(init_seg)); + if (!candidate) + return false; + memcpy(candidate, import_qn, import_len + 1); bool matched = false; for (;;) { pxc_module_entry_t *entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); @@ -2058,6 +2078,31 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, matched = true; break; } + if (caller_lang == CBM_LANG_PYTHON) { + size_t cand_len = strlen(candidate); + memcpy(candidate + cand_len, init_seg, sizeof(init_seg)); + entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); + candidate[cand_len] = '\0'; + /* A suffixed module may be a non-Python directory named __init__. + * Only this new probe filters strictly; normal-entry precedence + * above remains unchanged. Eligibility is independent of dedup. */ + bool eligible = false; + for (int j = 0; entry && j < entry->count; j++) { + int di = entry->indices[j]; + if (all_defs[di].lang != CBM_LANG_PYTHON) + continue; + eligible = true; + if (!selected[di]) { + selected[di] = true; + if (total) + (*total)++; + } + } + if (eligible) { + matched = true; + break; + } + } char *dot = strrchr(candidate, '.'); if (!dot) { break; @@ -2088,6 +2133,7 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, } } free(candidate); + return true; } CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) { @@ -2166,7 +2212,10 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef int total = 0; pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); for (int i = 0; i < imp_count; i++) { - pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total); + if (!pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total)) { + free(selected); + return NULL; /* out_success remains false: dispatcher uses all_defs */ + } } if (pxc_is_jvm_lang(caller_lang) && idx->namespace_ht) { const char *namespace_key = pxc_namespace_index_key(caller_namespace); diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 726199fde8..af415e13b3 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -573,9 +573,9 @@ static bool is_checked_exception(const char *name) { static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -2219,7 +2219,7 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou const CBMCall *call, const char *route_path, const char *handler_ref, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic) { + const char **ik, const char **iv, int ic, CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char rqn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -2238,7 +2238,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou esc_rp); cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); + cbm_resolution_t hres = + cbm_registry_resolve_lang(registry, handler_ref, module_qn, ik, iv, ic, lang); if (hres.qualified_name && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); if (h) { @@ -2531,7 +2532,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); const char *arg = call->first_string_arg; @@ -2558,7 +2559,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count); + registry, main_gbuf, imp_keys, imp_vals, imp_count, lang); return; } /* No path found — fall through to normal CALLS edge */ @@ -2996,8 +2997,8 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB lang, lsp->strategy, lsp->callee_qn, rc->project_name); if ((!res.qualified_name || !res.qualified_name[0]) && !call->requires_lsp_resolution && !rust_external) { - res = cbm_registry_resolve(rc->registry, call->callee_name, module_qn, imp_keys, - imp_vals, imp_count); + res = cbm_registry_resolve_lang(rc->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } atomic_fetch_add_explicit(&rc->time_ns_rc_resolve, extract_now_ns() - _rc_t0, memory_order_relaxed); @@ -3101,7 +3102,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB .strategy = "service_pattern"}; emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false); + imp_count, false, lang); continue; } } @@ -3118,7 +3119,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * self-call, so it keeps only the route/service edges. */ emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, rust_external); + imp_count, rust_external, lang); } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { /* Native `fetch()` (#856): only the global API once resolution * has failed to find a local/imported `fetch`. Call the low-level @@ -3170,7 +3171,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB if (url_or_topic) { emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false); + false, lang); ws->calls_resolved++; } } @@ -3179,7 +3180,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call); + drop_plain_call, lang); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; @@ -3245,8 +3246,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(rc->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(rc->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3295,7 +3296,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve throws/raises for one file. */ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *rel, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; if (!thr->exception_name || !thr->enclosing_func_qn) { @@ -3310,8 +3312,8 @@ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(rc->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + rc->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3337,8 +3339,8 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi if (!src) { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, rw->var_name, module_qn, imp_keys, - imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3365,12 +3367,14 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi /* Resolve base_classes → INHERITS edges for one definition. */ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->base_classes) { return; } for (int b = 0; def->base_classes[b]; b++) { - const char *bqn = resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic); + const char *bqn = + resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic, lang); if (!bqn) { continue; } @@ -3389,7 +3393,8 @@ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve decorators → DECORATES edges for one definition. */ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->decorators) { return; } @@ -3399,14 +3404,14 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws if (fn[0] == '\0') { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, fn, mq, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, fn, mq, ik, iv, ic, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(fn, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but * declared with an `Attribute` suffix (`class LogAttribute`). */ char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", fn); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(rc->registry, with_suffix, mq, ik, iv, ic); + res = cbm_registry_resolve_lang(rc->registry, with_suffix, mq, ik, iv, ic, lang); } } const cbm_gbuf_node_t *dn = NULL; @@ -3459,7 +3464,8 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws /* Resolve INHERITS + DECORATES + IMPLEMENTS for one file. */ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int d = 0; d < result->defs.count; d++) { CBMDefinition *def = &result->defs.items[d]; if (!def->qualified_name) { @@ -3469,8 +3475,8 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (!node) { continue; } - resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); - resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); + resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); } for (int t = 0; t < result->impl_traits.count; t++) { CBMImplTrait *it = &result->impl_traits.items[t]; @@ -3478,9 +3484,9 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *tqn = resolve_as_class(rc->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); const char *sqn = resolve_as_class(rc->registry, it->struct_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); if (!tqn || !sqn) { continue; } @@ -3763,7 +3769,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { (strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 || strcmp(result->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 || strcmp(result->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 || - strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0)) { + strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0)) { rc->lsp_failures[file_idx] = result->error_msg; ws->errors++; } @@ -3816,7 +3823,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── THROWS / RAISES ───────────────────────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_throws, extract_now_ns() - _ph_t0, memory_order_relaxed); @@ -3827,7 +3834,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, memory_order_relaxed); diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab00831..434ff769fd 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -177,9 +177,9 @@ static void free_import_map(const char **keys, const char **vals, int count) { /* Resolve a class/type name through the registry. Returns borrowed QN or NULL. */ static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -546,14 +546,14 @@ static void synth_decorator_qn(const char *func_name, char *out, size_t outsz) { /* Resolve one decorator and create DECORATES edge. */ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *node, const char *decorator, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, int *count) { + const char **imp_vals, int imp_count, int *count, CBMLanguage lang) { char func_name[CBM_SZ_256]; extract_decorator_func(decorator, func_name, sizeof(func_name)); if (func_name[0] == '\0') { return; } - cbm_resolution_t res = - cbm_registry_resolve(ctx->registry, func_name, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(func_name, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but declared * with the conventional `Attribute` suffix (`class LogAttribute`). Retry @@ -561,8 +561,8 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", func_name); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(ctx->registry, with_suffix, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, with_suffix, module_qn, imp_keys, + imp_vals, imp_count, lang); } } const cbm_gbuf_node_t *dec = NULL; @@ -613,7 +613,7 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, int *inherits_count, - int *decorates_count) { + int *decorates_count, CBMLanguage lang) { if (!def->qualified_name) { return; } @@ -624,7 +624,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->base_classes) { for (int b = 0; def->base_classes[b]; b++) { const char *base_qn = resolve_as_class(ctx->registry, def->base_classes[b], module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, lang); if (!base_qn) { continue; } @@ -646,7 +646,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->decorators) { for (int dc = 0; def->decorators[dc]; dc++) { resolve_decorator(ctx, node, def->decorators[dc], module_qn, imp_keys, imp_vals, - imp_count, decorates_count); + imp_count, decorates_count, lang); } } } @@ -683,12 +683,12 @@ static int resolve_impl_traits(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res continue; } const char *trait_qn = resolve_as_class(ctx->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, CBM_LANG_RUST); if (!trait_qn) { continue; } const char *struct_qn = resolve_as_class(ctx->registry, it->struct_name, module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, CBM_LANG_RUST); if (!struct_qn) { continue; } @@ -737,7 +737,7 @@ int cbm_pipeline_pass_semantic(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *f /* ── INHERITS + DECORATES from definitions ──────────────── */ for (int d = 0; d < result->defs.count; d++) { sem_process_def_edges(ctx, &result->defs.items[d], module_qn, imp_keys, imp_vals, - imp_count, &inherits_count, &decorates_count); + imp_count, &inherits_count, &decorates_count, files[i].language); } /* ── IMPLEMENTS from impl_traits (Rust) ─────────────────── */ diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 9738f7198d..096130775e 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -200,8 +200,8 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(ctx->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(ctx->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(ctx->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -248,7 +248,7 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res /* Resolve THROWS/RAISES edges for one file's extracted throws. */ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { int resolved = 0; for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; @@ -262,8 +262,8 @@ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + ctx->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); const cbm_gbuf_node_t *tgt = NULL; if (res.qualified_name && res.qualified_name[0]) { @@ -295,8 +295,8 @@ static int resolve_rw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result continue; } - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, rw->var_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -379,8 +379,8 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil usage_resolved += resolve_usage_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); - throw_resolved += - resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count); + throw_resolved += resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, + imp_count, files[i].language); rw_resolved += resolve_rw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae98..397691ee3d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -21,6 +21,15 @@ #include #include +/* Internal explicit-language resolution; public raw/lineage APIs stay raw. */ +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pxc_test_import_probe_fail_after(int successful_allocations); +#endif + /* ── Shared pipeline constants ─────────────────────────────────── */ /* Per-file tree-sitter parse budget, in MICROSECONDS of this thread's CPU time diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 1f5335c10c..bb8d19bdd1 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -24,7 +24,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #define REG_HALF_PENALTY 0.5 #define DEFAULT_CONFIDENCE 0.5 -#include "pipeline/pipeline.h" +#include "pipeline_internal.h" #include "cbm.h" /* cbm_label_is_relation — the resolve-time relation veto */ #include "foundation/compat.h" /* CBM_TLS */ #include "foundation/hash_table.h" @@ -387,6 +387,7 @@ void cbm_registry_import_map_cache_end(void) { * eliminated. */ typedef struct { cbm_resolution_t res; + size_t scope_len; /* raw and Python-scoped requests may share one file cache */ } resolve_cache_entry_t; static CBM_TLS CBMHashTable *_resolve_cache = NULL; @@ -1099,22 +1100,45 @@ static cbm_resolution_t resolve_import_map(const cbm_registry_t *r, const char * return empty_result(); } -/* Strategy 2: Same-module match */ -static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, - const char *suffix, const char *module_qn) { +/* Exact same-module lookup without ever hashing a truncated candidate. The + * long path uses the canonical short-name bucket, not heuristic name limits. */ +static const char *same_module_candidate(const cbm_registry_t *r, const char *module_qn, + size_t scope_len, const char *name) { + if (!module_qn || !name) + return NULL; + size_t name_len = strlen(name); char candidate[CBM_SZ_512]; - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, callee_name); - const char *stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } - if (suffix && suffix[0]) { - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, suffix); - stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } + if (scope_len < sizeof(candidate) && name_len < sizeof(candidate) - scope_len - 1) { + memcpy(candidate, module_qn, scope_len); + candidate[scope_len] = '.'; + memcpy(candidate + scope_len + 1, name, name_len + 1); + return cbm_ht_get_key(r->exact, candidate); + } + qn_array_t *arr = (qn_array_t *)cbm_ht_get(r->by_name, simple_name(name)); + if (!arr) + return NULL; + for (int i = 0; i < arr->count; i++) { + const char *qn = arr->items[i]; + if (!qn) + continue; + size_t qn_len = strlen(qn); + if (qn_len > scope_len && qn_len - scope_len - 1 == name_len && + memcmp(qn, module_qn, scope_len) == 0 && qn[scope_len] == '.' && + memcmp(qn + scope_len + 1, name, name_len) == 0) + return qn; } + return NULL; +} + +/* Strategy 2: Same-module match. Scoring elsewhere retains the raw file QN. */ +static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, + const char *suffix, const char *module_qn, + size_t scope_len) { + const char *stored_key = same_module_candidate(r, module_qn, scope_len, callee_name); + if (!stored_key && suffix && suffix[0]) + stored_key = same_module_candidate(r, module_qn, scope_len, suffix); + if (stored_key) + return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; return empty_result(); } @@ -1360,7 +1384,8 @@ static cbm_resolution_t resolve_name_lookup(const cbm_registry_t *r, const char * cbm_registry_resolve owns the per-file cache). */ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { + const char **import_map_vals, int import_map_count, + size_t scope_len) { /* Split callee at the first path separator: "pkg.Func" → prefix="pkg", * suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever * separator appears first ("lib::square" → prefix="lib", suffix="square"). @@ -1393,7 +1418,7 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch resolve_import_map(r, prefix, suffix, import_map_keys, import_map_vals, import_map_count); if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 2: same module */ - res = resolve_same_module(r, callee_name, suffix, module_qn); + res = resolve_same_module(r, callee_name, suffix, module_qn, scope_len); } if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 3+4: name lookup */ @@ -1402,9 +1427,10 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch return res; } -cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { +static cbm_resolution_t registry_resolve_scoped(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + size_t scope_len) { if (!r || !callee_name) { return empty_result(); } @@ -1412,16 +1438,13 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Per-file cache: same callee_name in N call sites → 1 chain walk * + N-1 O(1) hash hits. module_qn is constant per file so the * cache key only needs callee_name. */ - if (_resolve_cache) { - resolve_cache_entry_t *cached = - (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); - if (cached) { - return cached->res; - } - } + resolve_cache_entry_t *cached = + _resolve_cache ? (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name) : NULL; + if (cached && cached->scope_len == scope_len) + return cached->res; cbm_resolution_t res = registry_resolve_chain(r, callee_name, module_qn, import_map_keys, - import_map_vals, import_map_count); + import_map_vals, import_map_count, scope_len); /* Data relations (Table/View) are lineage-only registry members: common * table names (users, orders, config) collide with code identifiers across @@ -1437,10 +1460,15 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Cache the result (including empty — caching the negative answer * is just as valuable; same name asks the same question). */ - if (_resolve_cache) { + if (cached) { + /* Same key, different effective scope: reuse its owned entry/key. */ + cached->res = res; + cached->scope_len = scope_len; + } else if (_resolve_cache) { resolve_cache_entry_t *e = (resolve_cache_entry_t *)malloc(sizeof(*e)); if (e) { e->res = res; + e->scope_len = scope_len; char *kdup = strdup(callee_name); if (kdup) { cbm_ht_set(_resolve_cache, kdup, e); @@ -1452,6 +1480,23 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle return res; } +cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, module_qn ? strlen(module_qn) : 0); +} + +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang) { + size_t scope_len = lang == CBM_LANG_PYTHON ? cbm_fqn_symbol_scope_len(module_qn) + : (module_qn ? strlen(module_qn) : 0); + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, scope_len); +} + cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count) { @@ -1464,7 +1509,7 @@ cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const cha * it would poison one variant with the other's semantics. SQL files hold * few distinct relation refs, so the chain walk stays cheap. */ return registry_resolve_chain(r, callee_name, module_qn, import_map_keys, import_map_vals, - import_map_count); + import_map_count, module_qn ? strlen(module_qn) : 0); } /* ── Fuzzy Resolve ──────────────────────────────────────────────── */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ba220bc84b..a7080137b9 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8530,6 +8530,352 @@ TEST(pipeline_python_module_callers_parallel) { return python_module_callers_pipeline_case(true); } +/* Python file identity keeps .__init__; its exported symbols do not. */ +static bool python_init_local_inheritance(cbm_store_t *store, const char *project, + const char *module) { + char child_qn[768], parent_qn[768], wrong_qn[768]; + int cn = snprintf(child_qn, sizeof(child_qn), "%s.%s.BooleanField", project, module); + int pn = snprintf(parent_qn, sizeof(parent_qn), "%s.%s.Field", project, module); + int wn = snprintf(wrong_qn, sizeof(wrong_qn), "%s.other.Field", project); + if (cn <= 0 || (size_t)cn >= sizeof(child_qn) || pn <= 0 || (size_t)pn >= sizeof(parent_qn) || + wn <= 0 || (size_t)wn >= sizeof(wrong_qn)) + return false; + cbm_node_t child = {0}, parent = {0}, wrong = {0}; + bool ok = cbm_store_find_node_by_qn(store, project, child_qn, &child) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, parent_qn, &parent) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, wrong_qn, &wrong) == CBM_STORE_OK && + child.id > 0 && parent.id > 0 && wrong.id > 0 && parent.id != wrong.id && + child.label && parent.label && wrong.label && strcmp(child.label, "Class") == 0 && + strcmp(parent.label, "Class") == 0 && strcmp(wrong.label, "Class") == 0; + cbm_edge_t *edges = NULL; + int count = 0, local = 0, foreign = 0; + if (ok) { + ok = cbm_store_find_edges_by_source_type(store, child.id, "INHERITS", &edges, &count) == + CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + local += edges[i].target_id == parent.id; + foreign += edges[i].target_id == wrong.id; + } + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&child); + cbm_node_free_fields(&parent); + cbm_node_free_fields(&wrong); + return ok && local == 1 && foreign == 0; +} + +static int python_init_scope_pipeline_case(bool parallel, const char *failure_stage) { + static const char body[] = "from util import normalize\n" + "from other import Field as OtherField\n" + "def helper(value):\n return normalize(value)\n" + "class Field:\n" + " def clean(self, value):\n return helper(value)\n" + " def validate(self, value):\n return self.clean(value)\n" + "class BooleanField(Field):\n" + " def to_python(self, value):\n" + " self.validate(value)\n return helper(value)\n" + "def build(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const char consumer[] = "from pkg import BooleanField, helper\n" + "def consume(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } local_edges[] = { + {"Field.clean", "Method", "helper", "Function"}, + {"Field.validate", "Method", "Field.clean", "Method"}, + {"BooleanField.to_python", "Method", "Field.validate", "Method"}, + {"BooleanField.to_python", "Method", "helper", "Function"}, + {"build", "Function", "BooleanField.to_python", "Method"}, + {"build", "Function", "helper", "Function"}, + }; + char root[512], path[768]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-init-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python package fixture"); + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file( + root, "pkg/__init__.py", + failure_stage ? "class S:\n def m(self):\n return 1\n" + "def run():\n s = S()\n return s.m()\n" + : body); + } + if (written && !failure_stage) { + written = python_module_write_fixture_file(root, "plain.py", body) && + python_module_write_fixture_file(root, "consumer.py", consumer) && + python_module_write_fixture_file(root, "util.py", + "def normalize(value):\n return value\n") && + python_module_write_fixture_file(root, "other.py", "class Field:\n pass\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "init_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "# inert parallel routing file\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python package fixture"); + } + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + "CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"}; + char *saved[3] = {0}; + bool saved_ok = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + for (int i = 0; i < 3; i++) + free(saved[i]); + th_rmtree(root); + FAIL("failed to save Python package fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= failure_stage ? cbm_setenv(keys[2], failure_stage, 1) : cbm_unsetenv(keys[2]); + n = snprintf(path, sizeof(path), "%s/package.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0, scope_errors = 0; + bool precise = true; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + for (int i = 0; i < error_count; i++) { + bool is_package = errors[i].path && strcmp(errors[i].path, "pkg/__init__.py") == 0; + if (failure_stage && is_package) { + scope_errors++; + const char *phase = strcmp(failure_stage, "raw") == 0 ? "extract" : "lsp_skipped"; + precise = precise && errors[i].phase && strcmp(errors[i].phase, phase) == 0 && + errors[i].reason && strcmp(errors[i].reason, CBM_PY_LSP_SCOPE_ERROR) == 0; + } else { + precise = false; + } + } + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + int file_count = 0; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (failure_stage ? 1 : 5) + (parallel ? 52 : 0); + cbm_store_free_nodes(files, file_count); + if (project && !failure_stage) { + const char *modules[] = {"plain", "pkg"}; + for (int m = 0; m < 2; m++) { + for (size_t i = 0; i < sizeof(local_edges) / sizeof(local_edges[0]); i++) { + char caller[128], target[128]; + snprintf(caller, sizeof(caller), "%s.%s", modules[m], local_edges[i].caller); + snprintf(target, sizeof(target), "%s.%s", modules[m], local_edges[i].target); + bool edge = python_module_exact_lsp_edge(store, project, caller, + local_edges[i].caller_label, target, + local_edges[i].target_label); + correct = correct && edge; + } + char caller[128]; + snprintf(caller, sizeof(caller), "%s.helper", modules[m]); + bool imported = python_module_exact_lsp_edge(store, project, caller, "Function", + "util.normalize", "Function"); + bool inherits = python_init_local_inheritance(store, project, modules[m]); + correct = correct && imported && inherits; + } + bool method = + python_module_exact_lsp_edge(store, project, "consumer.consume", "Function", + "pkg.BooleanField.to_python", "Method"); + bool function = python_module_exact_lsp_edge(store, project, "consumer.consume", + "Function", "pkg.helper", "Function"); + correct = correct && method && function; + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, failure_stage ? 1 : 0); + ASSERT_EQ(scope_errors, failure_stage ? 1 : 0); + ASSERT_TRUE(precise && correct); + PASS(); +} + +TEST(pipeline_python_init_scope_sequential) { + return python_init_scope_pipeline_case(false, NULL); +} +TEST(pipeline_python_init_scope_parallel) { + return python_init_scope_pipeline_case(true, NULL); +} +TEST(pipeline_python_init_raw_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "raw"); +} +TEST(pipeline_python_init_raw_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "raw"); +} +TEST(pipeline_python_init_cross_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "cross"); +} +TEST(pipeline_python_init_cross_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "cross"); +} +TEST(pipeline_python_init_register_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "register"); +} +TEST(pipeline_python_init_register_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "register"); +} + +/* Compare identities, not just counts; a duplicate import must stop at the + * same nearest eligible entry even when it contributes no new selections. */ +static bool python_init_filter_is(CBMLSPDef *defs, int def_count, const char *const *imports, + int import_count, const char *const *expected, int count) { + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, def_count); + int got_count = -1; + bool succeeded = false; + CBMLSPDef *got = + cbm_pxc_filter_defs_for_file(index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", imports, + import_count, &got_count, &succeeded); + bool correct = index && succeeded && got_count == count && (count == 0 || got); + for (int i = 0; correct && i < count; i++) { + int hits = 0; + for (int j = 0; j < got_count; j++) + hits += got[j].qualified_name && strcmp(got[j].qualified_name, expected[i]) == 0; + correct = hits == 1; + } + free(got); + cbm_pxc_free_module_def_index(index); + return correct; +} + +TEST(pipeline_python_init_import_filter_precedence) { + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Foreign", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.Parent", + .def_module_qn = "test.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.foreign.Only", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.unknown.Only", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1}, + {.qualified_name = "test.plain.Exact", + .def_module_qn = "test.plain", + .lang = CBM_LANG_JAVA}, + {.qualified_name = "test.plain.Package", + .def_module_qn = "test.plain.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.nested.Item", + .def_module_qn = "test.pkg.nested.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Item", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + }; + const char *pkg[] = {"test.pkg", "test.pkg.Store", "test.pkg.Store"}; + const char *want_pkg[] = {"test.consumer.use", "test.pkg.Store"}; + const char *foreign[] = {"test.foreign.Only", "test.unknown.Only"}; + const char *want_parent[] = {"test.consumer.use", "test.Parent"}; + const char *normal[] = {"test.plain.Exact", "test.plain.Exact"}; + const char *want_normal[] = {"test.consumer.use", "test.plain.Exact"}; + const char *nested[] = {"test.pkg.nested.Item", "test.pkg.nested.Item"}; + const char *want_nested[] = {"test.consumer.use", "test.pkg.nested.Item"}; + const char *submodule[] = {"test.pkg.sub.Item"}; + const char *want_submodule[] = {"test.consumer.use", "test.pkg.sub.Item"}; + int n = (int)(sizeof(defs) / sizeof(defs[0])); + bool pkg_ok = python_init_filter_is(defs, n, pkg, 3, want_pkg, 2); + bool foreign_ok = python_init_filter_is(defs, n, foreign, 2, want_parent, 2); + bool normal_ok = python_init_filter_is(defs, n, normal, 2, want_normal, 2); + bool nested_ok = python_init_filter_is(defs, n, nested, 2, want_nested, 2); + bool submodule_ok = python_init_filter_is(defs, n, submodule, 1, want_submodule, 2); + ASSERT_TRUE(pkg_ok && foreign_ok && normal_ok && nested_ok && submodule_ok); + PASS(); +} + +TEST(pipeline_python_init_import_probe_failure_falls_back) { + const char *source = "from pkg import Store\n" + "def use():\n store = Store()\n return store.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .short_name = "use", + .label = "Function", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *keys[] = {"Store"}; + const char *values[] = {"test.pkg.Store"}; + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, 3); + int count = -1; + bool succeeded = true; + cbm_pxc_test_import_probe_fail_after(0); + CBMLSPDef *filtered = cbm_pxc_filter_defs_for_file( + index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", values, 1, &count, &succeeded); + cbm_pxc_test_import_probe_fail_after(-1); + bool failed_precisely = index && !filtered && count == 0 && !succeeded; + free(filtered); + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_pxc_test_import_probe_fail_after(0); + cbm_pxc_dispatch_file(CBM_LANG_PYTHON, &result, source, (int)strlen(source), "consumer.py", + "test.consumer", NULL, index, defs, 3, keys, values, 1, NULL, NULL); + cbm_pxc_test_import_probe_fail_after(-1); + const char *site = strstr(source, "store.get()"); + int exact = 0; + for (int i = 0; i < result.resolved_calls.count; i++) { + const CBMResolvedCall *call = &result.resolved_calls.items[i]; + if (call->caller_qn && strcmp(call->caller_qn, "test.consumer.use") == 0 && + call->callee_qn && strcmp(call->callee_qn, "test.pkg.Store.get") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && call->strategy && + strncmp(call->strategy, "lsp_", 4) == 0 && site && + call->site_start_byte == (uint32_t)(site - source) && + call->site_end_byte == (uint32_t)(site - source + strlen("store.get()"))) + exact++; + } + bool resolved = !result.has_error && !result.lsp_skipped && exact == 1; + cbm_arena_destroy(&result.arena); + cbm_pxc_free_module_def_index(index); + ASSERT_TRUE(failed_precisely && resolved); + PASS(); +} + /* #1277: typed instance fields of a Python class imported from another file. * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an * LSP strategy. */ @@ -16776,6 +17122,16 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { SUITE(pipeline) { RUN_TEST(pipeline_python_module_callers_sequential); RUN_TEST(pipeline_python_module_callers_parallel); + RUN_TEST(pipeline_python_init_scope_sequential); + RUN_TEST(pipeline_python_init_scope_parallel); + RUN_TEST(pipeline_python_init_raw_scope_failure_sequential); + RUN_TEST(pipeline_python_init_raw_scope_failure_parallel); + RUN_TEST(pipeline_python_init_cross_scope_failure_sequential); + RUN_TEST(pipeline_python_init_cross_scope_failure_parallel); + RUN_TEST(pipeline_python_init_register_scope_failure_sequential); + RUN_TEST(pipeline_python_init_register_scope_failure_parallel); + RUN_TEST(pipeline_python_init_import_filter_precedence); + RUN_TEST(pipeline_python_init_import_probe_failure_falls_back); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index adc84913b3..b16f5f87a8 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -2913,7 +2913,403 @@ TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites) { PASS(); } +/* Package symbols use their parent scope while Module callers retain the file. */ +TEST(pylsp_init_scope_context_and_helper) { + static const struct { + const char *raw; + const char *scope; + } cases[] = { + {"test.__init__", "test"}, + {"test.pkg.__init__", "test.pkg"}, + {"test.pkg.nested.__init__", "test.pkg.nested"}, + {"test.plain", "test.plain"}, + {"test.x__init__", "test.x__init__"}, + {"__init__", "__init__"}, + {"test.index", "test.index"}, + {NULL, NULL}, + }; + bool correct = true; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry registry; + cbm_registry_init(®istry, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, "pass\n", 5, ®istry, cases[i].raw, &out); + correct = + correct && ctx.eval_failure == CBM_LSP_COMPLETE && ctx.current_scope && + ctx.file_module_qn == cases[i].raw && + cbm_fqn_symbol_scope_len(cases[i].raw) == (cases[i].scope ? strlen(cases[i].scope) : 0); + correct = + correct && (cases[i].scope ? ctx.module_qn && strcmp(ctx.module_qn, cases[i].scope) == 0 + : ctx.module_qn == NULL); + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations) { + static const char source[] = + "class Store:\n def get(self):\n return 1\n" + "def identity(value: Store) -> Store:\n return value\n" + "def quoted(value: 'Store') -> 'Store':\n return value\n" + "def wrapped(values: list[Store]) -> Store:\n return values[0]\n" + "def run(first: Store, second: 'Store', values: list[Store]):\n" + " first.get()\n second.get()\n values[0].get()\n" + " identity(first).get()\n quoted(second).get()\n" + " wrapped(values).get()\n" + "class User:\n def use(self, item: Store):\n return item.get()\n" + "top = Store()\ntop.get()\n"; + static const char *paths[] = {"plain.py", "__init__.py", "pkg/nested/__init__.py"}; + static const char *scopes[] = {"test.plain", "test", "test.pkg.nested"}; + static const char *modules[] = {"test.plain", "test.__init__", "test.pkg.nested.__init__"}; + static const char *sites[] = {"first.get()", "second.get()", + "values[0].get()", "identity(first).get()", + "quoted(second).get()", "wrapped(values).get()"}; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[mode], 0, NULL, NULL); + bool extracted = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + r->module_qn && strcmp(r->module_qn, modules[mode]) == 0; + correct = correct && extracted; + if (extracted) { + char caller[128], target[128], method[128]; + snprintf(caller, sizeof(caller), "%s.run", scopes[mode]); + snprintf(target, sizeof(target), "%s.Store.get", scopes[mode]); + snprintf(method, sizeof(method), "%s.User.use", scopes[mode]); + correct = correct && pylsp_module_has_def(r, caller) && + pylsp_module_has_def(r, target) && pylsp_module_has_def(r, method); + for (size_t j = 0; j < sizeof(sites) / sizeof(sites[0]); j++) { + bool joined = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + sites[j], caller, target); + correct = correct && joined; + } + bool method_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "item.get()", method, target); + bool module_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "top.get()", modules[mode], target); + correct = correct && method_join && module_join; + } + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +static bool pylsp_init_named_type(const CBMType *type, const char *qn) { + return type && type->kind == CBM_TYPE_NAMED && type->data.named.qualified_name && + strcmp(type->data.named.qualified_name, qn) == 0; +} + +static bool pylsp_init_signature(const CBMTypeRegistry *registry, const char *function, + const char *expected_type) { + const CBMRegisteredFunc *f = cbm_registry_lookup_func(registry, function); + const CBMType *sig = f ? f->signature : NULL; + if (!sig || sig->kind != CBM_TYPE_FUNC || !sig->data.func.param_types) + return false; + const CBMType **params = sig->data.func.param_types; + if (!pylsp_init_named_type(params[0], expected_type) || + !pylsp_init_named_type(params[1], expected_type) || !params[2] || params[3]) + return false; + const CBMType *container = params[2]; + return container->kind == CBM_TYPE_TEMPLATE && container->data.template_type.template_name && + strcmp(container->data.template_type.template_name, "list") == 0 && + container->data.template_type.arg_count == 1 && + container->data.template_type.template_args && + pylsp_init_named_type(container->data.template_type.template_args[0], expected_type); +} + +TEST(pylsp_init_scope_cross_signatures_keep_definition_language) { + const char *params[] = {"Token", "'Token'", "list[Token]"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.foreign.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.unknown.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1, + .signature_param_types = params, + .signature_param_count = 3}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry fallback; + cbm_registry_init(&fallback, &arena); + bool registered = cbm_py_lsp_test_register_defs(&arena, &fallback, defs, 3); + bool correct = registered && + pylsp_init_signature(&fallback, "test.pkg.call", "test.pkg.Token") && + pylsp_init_signature(&fallback, "test.foreign.__init__.call", + "test.foreign.__init__.Token") && + pylsp_init_signature(&fallback, "test.unknown.__init__.call", + "test.unknown.__init__.Token"); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && base && base->read_only && + pylsp_init_signature(base, "test.pkg.call", "test.pkg.Token") && + !cbm_registry_lookup_func(base, "test.foreign.__init__.call") && + !cbm_registry_lookup_func(base, "test.unknown.__init__.call"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_external_classes_and_cross_exact_joins) { + const char *source = "from pkg.sub import Widget as Imported\n" + "def use():\n store = Store()\n store.get()\n" + " imported = Imported()\n imported.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget", + .short_name = "Widget", + .label = "Class", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.sub", + .receiver_type = "test.pkg.sub.Widget", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Outer.Nested", + .short_name = "Nested", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + }; + const char *names[] = {"Imported"}; + const char *qns[] = {"test.pkg.sub.Widget"}; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + TSParser *parser = ts_parser_new(); + if (parser) + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + parser ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && tree && + pylsp_module_has_def(r, "test.pkg.use"); + for (int prebuilt = 0; correct && prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 5); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &overlay, "test.pkg.__init__", &out); + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool globals = base && ctx.eval_failure == CBM_LSP_COMPLETE && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "Store"), + "test.pkg.Store") && + !cbm_scope_lookup_local(ctx.current_scope, "Widget") && + !cbm_scope_lookup_local(ctx.current_scope, "Nested"); + char module_input[] = "test.pkg.__init__"; + CBMLSPStatus status = + prebuilt + ? cbm_run_py_lsp_cross_with_registry_status(&arena, source, (int)strlen(source), + module_input, &overlay, names, qns, 1, + tree, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), module_input, + defs, 5, names, qns, 1, tree, &out, NULL); + memset(module_input, 'x', sizeof(module_input) - 1); + bool store = pylsp_module_site_joins(&r->calls, &out, source, "store.get()", "test.pkg.use", + "test.pkg.Store.get"); + bool imported = pylsp_module_site_joins(&r->calls, &out, source, "imported.get()", + "test.pkg.use", "test.pkg.sub.Widget.get"); + correct = correct && globals && status == CBM_LSP_COMPLETE && store && imported; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers) { + const char *source = "def run():\n value = Store()\n return value.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + cbm_py_lsp_test_scope_fail_after(0); + CBMFileResult *raw = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && + raw->resolved_calls.count == 0 && pylsp_module_has_def(raw, "test.pkg.run"); + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 2); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCall prior = {.caller_qn = "completed.caller", + .callee_qn = "completed.target", + .strategy = "prior.strategy", + .reason = "prior.reason", + .confidence = 0.125f, + .kind = CBM_RESOLVED_INVOCATION, + .site_start_byte = 7, + .site_end_byte = 14}; + unsigned char saved[sizeof(prior)]; + memcpy(saved, &prior, sizeof(prior)); + CBMCall previous_synthetic = {.callee_name = "completed.synthetic"}; + CBMCallArray synthetic = {.items = &previous_synthetic, .count = 1, .cap = 1}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + cbm_py_lsp_test_scope_fail_after(mode == 2 ? 1 : 0); + /* mode 2 gets past context normalization, then fails signature scope. */ + CBMLSPStatus status = mode == 1 + ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), "test.pkg.__init__", + &overlay, NULL, NULL, 0, NULL, &out, &synthetic) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + "test.pkg.__init__", defs, 2, NULL, + NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && base && status == CBM_LSP_SCOPE_FAILED && out.count == 1 && + out.items == &prior && memcmp(saved, &prior, sizeof(prior)) == 0 && + synthetic.count == 1 && synthetic.items == &previous_synthetic && + strcmp(previous_synthetic.callee_name, "completed.synthetic") == 0; + cbm_py_lsp_test_scope_fail_after(0); + bool complete = + mode == 1 + ? cbm_run_py_lsp_cross_with_registry(&arena, source, (int)strlen(source), + "test.pkg.__init__", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.pkg.__init__", + defs, 2, NULL, NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = + correct && !complete && out.count == 1 && memcmp(saved, &prior, sizeof(prior)) == 0; + CBMBatchPyLSPFile file = {.source = source, + .source_len = (int)strlen(source), + .module_qn = "test.pkg.__init__", + .defs = defs, + .def_count = 2}; + CBMResolvedCallArray batch = {0}; + cbm_py_lsp_test_scope_fail_after(0); + bool batch_complete = cbm_batch_py_lsp_cross(&arena, &file, 1, &batch); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && !batch_complete && batch.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback) { + const char *source = "def use():\n obj = Item()\n return obj.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.plain.Item", + .short_name = "Item", + .label = "Class", + .def_module_qn = "test.plain", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.plain.Item.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.plain", + .receiver_type = "test.plain.Item", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.unused.work", + .short_name = "work", + .label = "Function", + .def_module_qn = "test.unused.__init__", + .lang = CBM_LANG_PYTHON}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + char *sentinel = cbm_arena_strdup(&shared, "caller-owned shared data"); + cbm_py_lsp_test_scope_fail_after(0); + CBMTypeRegistry *failed = cbm_py_build_cross_registry(&shared, defs, 3); + CBMResolvedCallArray out = {0}; + CBMLSPStatus fallback = + cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), "test.plain", defs, 2, + NULL, NULL, 0, NULL, &out, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + bool correct = !failed && sentinel && strcmp(sentinel, "caller-owned shared data") == 0 && + fallback == CBM_LSP_COMPLETE; + CBMFileResult *extracted = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "test", "plain.py", 0, NULL, NULL); + correct = correct && extracted && !extracted->has_error && !extracted->parse_incomplete && + pylsp_module_has_def(extracted, "test.plain.use") && + pylsp_module_site_joins(&extracted->calls, &out, source, "obj.get()", + "test.plain.use", "test.plain.Item.get"); + CBMTypeRegistry *fresh = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && fresh && fresh->read_only && + cbm_registry_lookup_func(fresh, "test.unused.work") && + strcmp(sentinel, "caller-owned shared data") == 0; + if (extracted) + cbm_free_result(extracted); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_init_scope_context_and_helper); + RUN_TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations); + RUN_TEST(pylsp_init_scope_cross_signatures_keep_definition_language); + RUN_TEST(pylsp_init_scope_external_classes_and_cross_exact_joins); + RUN_TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers); + RUN_TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback); + RUN_TEST(pylsp_module_callers_join_exact_raw_occurrences); RUN_TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair); RUN_TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites); diff --git a/tests/test_registry.c b/tests/test_registry.c index 278e97d9b0..4d305356c4 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1288,7 +1288,277 @@ TEST(resolve_import_map_alias_with_suffix_hits_method) { PASS(); } +/* Python package scope is an explicit resolver input, not a property inferred + * from a QN shared by other languages. Keep these fixtures separate from the + * raw public-API tests above. */ +static bool registry_scope_match(cbm_resolution_t result, const char *qn, const char *strategy) { + return qn && strategy && result.qualified_name && result.strategy && + strcmp(result.qualified_name, qn) == 0 && strcmp(result.strategy, strategy) == 0 && + result.candidate_count == 1; +} + +TEST(python_symbol_scope_len_preserves_file_identity) { + const char *inputs[] = {"proj.pkg.__init__", + "proj.__init__", + "proj.pkg.__init__.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + NULL}; + const char *scopes[] = {"proj.pkg", + "proj", + "proj.pkg.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + ""}; + bool lengths = true; + for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); i++) { + lengths = cbm_fqn_symbol_scope_len(inputs[i]) == strlen(scopes[i]) && lengths; + } + char *module = cbm_pipeline_fqn_module("proj", "pkg/__init__.py"); + char *symbol = cbm_pipeline_fqn_compute("proj", "pkg/__init__.py", "Field"); + bool identities = module && symbol && strcmp(module, "proj.pkg.__init__") == 0 && + strcmp(symbol, "proj.pkg.Field") == 0 && + cbm_fqn_symbol_scope_len(module) == strlen("proj.pkg"); + free(module); + free(symbol); + ASSERT_TRUE(lengths && identities); + PASS(); +} + +TEST(registry_python_package_scope_is_explicit) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.forms.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.__init__.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.plain.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.index.Field", "Class"); + const char *keys[] = {"forms"}; + const char *vals[] = {"proj.forms"}; + const char *module = "proj.db.fields.__init__"; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t package = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, CBM_LANG_PYTHON); + bool python = registry_scope_match(package, "proj.db.fields.Field", "same_module"); + cbm_resolution_t plain = + cbm_registry_resolve_lang(r, "Field", "proj.db.plain", keys, vals, 1, CBM_LANG_PYTHON); + bool normal = registry_scope_match(plain, "proj.db.plain.Field", "same_module"); + cbm_resolution_t raw = cbm_registry_resolve(r, "Field", module, keys, vals, 1); + bool preserved = registry_scope_match(raw, "proj.db.fields.__init__.Field", "same_module"); + const CBMLanguage raw_languages[] = {CBM_LANG_JAVA, CBM_LANG_GO, CBM_LANG_JAVASCRIPT, + (CBMLanguage)-1, CBM_LANG_COUNT}; + for (size_t i = 0; i < sizeof(raw_languages) / sizeof(raw_languages[0]); i++) { + cbm_resolution_t result = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, raw_languages[i]); + preserved = registry_scope_match(result, raw.qualified_name, "same_module") && + result.confidence == raw.confidence && preserved; + } + cbm_resolution_t index = + cbm_registry_resolve_lang(r, "Field", "proj.db.index", keys, vals, 1, CBM_LANG_JAVASCRIPT); + bool index_raw = registry_scope_match(index, "proj.db.index.Field", "same_module"); + cbm_registry_free(r); + ASSERT_TRUE(seeded && python && normal && preserved && index_raw); + PASS(); +} + +TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.vendor.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.__init__.Field", "Class"); + cbm_registry_add(r, "choose", "proj.pkg.__init__.child.choose", "Function"); + cbm_registry_add(r, "choose", "proj.pkg.peer.choose", "Function"); + const char *keys[] = {"Field"}; + const char *vals[] = {"proj.vendor.Field"}; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t imported = + cbm_registry_resolve_lang(r, "Field", "proj.pkg.__init__", keys, vals, 1, CBM_LANG_PYTHON); + bool precedence = registry_scope_match(imported, "proj.vendor.Field", "import_map"); + cbm_resolution_t raw = cbm_registry_resolve(r, "choose", "proj.pkg.__init__", NULL, NULL, 0); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, "choose", "proj.pkg.__init__", NULL, NULL, 0, CBM_LANG_PYTHON); + bool scoring = + raw.qualified_name && python.qualified_name && raw.strategy && python.strategy && + strcmp(raw.qualified_name, "proj.pkg.__init__.child.choose") == 0 && + strcmp(python.qualified_name, raw.qualified_name) == 0 && + strcmp(raw.strategy, "suffix_match") == 0 && strcmp(python.strategy, raw.strategy) == 0 && + python.confidence == raw.confidence && python.candidate_count == raw.candidate_count; + cbm_registry_free(r); + ASSERT_TRUE(seeded && precedence && scoring); + PASS(); +} + +TEST(registry_python_scope_cache_modes_and_raw_lineage) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Entry", "proj.pkg.Entry", "Function"); + cbm_registry_add(r, "Entry", "proj.pkg.__init__.Entry", "Function"); + cbm_registry_add(r, "Reject", "proj.pkg.Reject", "Table"); + cbm_registry_add(r, "Reject", "proj.pkg.__init__.Reject", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.Accept", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.__init__.Accept", "Table"); + cbm_registry_add(r, "Relation", "proj.pkg.Relation", "View"); + cbm_registry_add(r, "Relation", "proj.pkg.__init__.Relation", "Table"); + bool seeded = cbm_registry_size(r) == 8; + bool correct = true; + /* Each cache lifetime has one fixed module/import map. Alternate modes in + * both starting orders; repeated same-mode requests also exercise hits. */ + for (int first_python = 0; first_python < 2; first_python++) { + cbm_registry_resolve_cache_begin(8); + for (int turn = 0; turn < 16; turn++) { + bool python = ((turn / 2 + first_python) % 2) != 0; + const char *names[] = {"Entry", "Reject", "Accept", "Relation", "Missing"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + cbm_resolution_t result = + python ? cbm_registry_resolve_lang(r, names[i], "proj.pkg.__init__", NULL, NULL, + 0, CBM_LANG_PYTHON) + : cbm_registry_resolve(r, names[i], "proj.pkg.__init__", NULL, NULL, 0); + const char *expected = NULL; + if (i == 0) { + expected = python ? "proj.pkg.Entry" : "proj.pkg.__init__.Entry"; + } else if (i == 1 && !python) { + expected = "proj.pkg.__init__.Reject"; + } else if (i == 2 && python) { + expected = "proj.pkg.Accept"; + } + correct = + (expected ? registry_scope_match(result, expected, "same_module") + : result.qualified_name == NULL && result.candidate_count == 0) && + correct; + } + /* The cached relation veto must never poison the uncached SQL API, + * nor may the SQL result bypass the next default cached veto. */ + cbm_resolution_t lineage = + cbm_registry_resolve_lineage(r, "Relation", "proj.pkg.__init__", NULL, NULL, 0); + correct = registry_scope_match(lineage, "proj.pkg.__init__.Relation", "same_module") && + correct; + cbm_resolution_t veto = cbm_registry_resolve_lang(r, "Relation", "proj.pkg.__init__", + NULL, NULL, 0, CBM_LANG_PYTHON); + correct = veto.qualified_name == NULL && veto.candidate_count == 0 && correct; + } + cbm_registry_resolve_cache_end(); + } + cbm_registry_free(r); + ASSERT_TRUE(seeded && correct); + PASS(); +} + +static const char *registry_scope_canonical(cbm_registry_t *r, const char *name, const char *qn) { + const char **candidates = NULL; + int count = 0; + cbm_registry_find_by_name(r, name, &candidates, &count); + for (int i = 0; i < count; i++) { + if (strcmp(candidates[i], qn) == 0) { + return candidates[i]; + } + } + return NULL; +} + +TEST(registry_python_scope_complete_candidate_boundaries) { + const size_t lengths[] = {511, 512, 513, 1536}; + bool complete = true; + for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { + cbm_registry_t *r = cbm_registry_new(); + if (!r) { + complete = false; + break; + } + char scope[2048]; + char module[2048]; + char target[2048]; + const char *name = "LongTarget"; + size_t scope_len = lengths[i] - strlen(name) - 1; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'a', scope_len - 5); + scope[scope_len] = '\0'; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + int tn = snprintf(target, sizeof(target), "%s.%s", scope, name); + bool shape = mn > 0 && (size_t)mn < sizeof(module) && tn > 0 && (size_t)tn == lengths[i] && + (size_t)tn < sizeof(target); + cbm_registry_add(r, name, target, "Function"); + if (lengths[i] > 511) { + char truncated[512]; + memcpy(truncated, target, sizeof(truncated) - 1); + truncated[sizeof(truncated) - 1] = '\0'; + cbm_registry_add(r, "truncated_decoy", truncated, "Function"); + } + const char *canonical = registry_scope_canonical(r, name, target); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, name, module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t normal = cbm_registry_resolve(r, name, scope, NULL, NULL, 0); + complete = shape && canonical && python.qualified_name == canonical && + normal.qualified_name == canonical && + registry_scope_match(python, target, "same_module") && + registry_scope_match(normal, target, "same_module") && complete; + cbm_registry_free(r); + } + ASSERT_TRUE(complete); + PASS(); +} + +TEST(registry_python_scope_qualified_terminal_bucket_uncapped) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + /* The exact targets are inserted after the heuristic limit. A long exact + * comparison must use the canonical terminal bucket, not "Owner.perform" + * as the bucket key, and must not stop at 256 candidates. */ + bool shape = true; + for (int i = 0; i < 300; i++) { + char qn[80]; + int n = snprintf(qn, sizeof(qn), "proj.noise%d.perform", i); + shape = n > 0 && (size_t)n < sizeof(qn) && shape; + cbm_registry_add(r, "perform", qn, "Method"); + } + char scope[1024]; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'b', 900); + scope[905] = '\0'; + char module[1024]; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + shape = mn > 0 && (size_t)mn < sizeof(module) && shape; + const char *names[] = {"Owner.perform", "Owner::perform"}; + bool resolved = true; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char target[1024]; + char prefixed[80]; + int tn = snprintf(target, sizeof(target), "%s.%s", scope, names[i]); + int pn = snprintf(prefixed, sizeof(prefixed), "receiver.%s", names[i]); + shape = tn > 0 && (size_t)tn < sizeof(target) && pn > 0 && (size_t)pn < sizeof(prefixed) && + shape; + cbm_registry_add(r, "perform", target, "Method"); + const char *canonical = registry_scope_canonical(r, "perform", target); + cbm_resolution_t direct = + cbm_registry_resolve_lang(r, names[i], module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t suffix = + cbm_registry_resolve_lang(r, prefixed, module, NULL, NULL, 0, CBM_LANG_PYTHON); + resolved = canonical && direct.qualified_name == canonical && + suffix.qualified_name == canonical && + registry_scope_match(direct, target, "same_module") && + registry_scope_match(suffix, target, "same_module") && resolved; + } + bool seeded = cbm_registry_size(r) == 302; + cbm_registry_free(r); + ASSERT_TRUE(shape && seeded && resolved); + PASS(); +} + SUITE(registry) { + RUN_TEST(python_symbol_scope_len_preserves_file_identity); + RUN_TEST(registry_python_package_scope_is_explicit); + RUN_TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring); + RUN_TEST(registry_python_scope_cache_modes_and_raw_lineage); + RUN_TEST(registry_python_scope_complete_candidate_boundaries); + RUN_TEST(registry_python_scope_qualified_terminal_bucket_uncapped); /* FQN */ RUN_TEST(fqn_simple); RUN_TEST(fqn_no_name); From 98177aa129a6f1a93c5e612b3620896725ae704a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 21:41:59 +0200 Subject: [PATCH 6/9] fix(python): resolve class-body and decorator calls Signed-off-by: Martin Vogel --- internal/cbm/lsp/py_lsp.c | 242 ++++++++++++++++++- internal/cbm/lsp/py_lsp.h | 11 +- tests/test_pipeline.c | 149 ++++++++++++ tests/test_py_lsp.c | 475 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 863 insertions(+), 14 deletions(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 29cec01e73..0c8571bc2d 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -40,6 +40,10 @@ static _Thread_local int py_test_memo_allocations_left = -1; static _Thread_local bool py_test_depth_failure; static _Thread_local int py_test_scope_allocations_left = -1; +static _Thread_local int py_test_class_frames_left = -1; +void cbm_py_lsp_test_class_frame_fail_after(int successful_allocations) { + py_test_class_frames_left = successful_allocations; +} void cbm_py_lsp_test_scope_fail_after(int successful_allocations) { py_test_scope_allocations_left = successful_allocations; } @@ -223,6 +227,7 @@ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node); static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node); static void py_process_statement(PyLSPContext *ctx, TSNode node); +static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node); static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int depth); static const CBMRegisteredFunc *py_lookup_attribute(PyLSPContext *ctx, const char *type_qn, const char *member_name); @@ -284,6 +289,25 @@ static CBMScope *py_scope_push_checked(PyLSPContext *ctx) { return child; } +/* New class execution frames must never fall back to their parent: body + * bindings would escape and subsequent calls could use those stale bindings. */ +static CBMScope *py_class_frame_push(PyLSPContext *ctx, CBMScope *parent) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_test_class_frames_left == 0) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return NULL; + } + if (py_test_class_frames_left > 0) + py_test_class_frames_left--; +#endif + CBMScope *frame = cbm_scope_push(ctx->arena, parent); + if (!frame || frame == parent) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return NULL; + } + return frame; +} + static void py_scope_clear_callable(PyLSPContext *ctx, const char *name) { if (cbm_scope_update_callable(ctx->current_scope, name, NULL)) { ctx->type_cache_gen++; @@ -2449,13 +2473,16 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { } else { py_scope_bind(ctx, name, rhs_type); } + // Class attributes must not replace file-wide dispatch entries. // Lambda registry: `fn = lambda x: ...`. - if (!ts_node_is_null(right) && strcmp(ts_node_type(right), "lambda") == 0) { + if (!ctx->class_body_outer_scope && !ts_node_is_null(right) && + strcmp(ts_node_type(right), "lambda") == 0) { py_register_lambda(ctx, name, right); } // Dict literal dispatch table: `funcs = {"a": foo, "b": bar}` // with all values being known function QNs. - if (!ts_node_is_null(right) && strcmp(ts_node_type(right), "dictionary") == 0) { + if (!ctx->class_body_outer_scope && !ts_node_is_null(right) && + strcmp(ts_node_type(right), "dictionary") == 0) { py_register_dict_literal(ctx, name, right); } } @@ -2663,7 +2690,11 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Walk the lambda body for any nested call sites with the params // bound to the call's arg types. Resolved calls get caller_qn // set to a synthetic child of the enclosing function. - TSNode lambda_node = py_lookup_lambda(ctx, fname); + /* These legacy entries have no lexical frame identity. During class + * execution, leave lambda replay unresolved instead of bypassing a + * local shadow or running the lambda against class locals. */ + TSNode lambda_node = + ctx->class_body_outer_scope ? (TSNode){0} : py_lookup_lambda(ctx, fname); if (!ts_node_is_null(lambda_node)) { TSNode params = ts_node_child_by_field_name(lambda_node, "parameters", 10); TSNode body = ts_node_child_by_field_name(lambda_node, "body", 4); @@ -2774,7 +2805,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Subscript-as-call: `funcs["a"]()` where funcs is a registered // dict-literal dispatch table. - if (strcmp(fk, "subscript") == 0) { + if (!ctx->class_body_outer_scope && strcmp(fk, "subscript") == 0) { TSNode container = ts_node_child_by_field_name(fn, "value", 5); TSNode key = ts_node_child_by_field_name(fn, "subscript", 9); if (!ts_node_is_null(container) && !ts_node_is_null(key) && @@ -3349,6 +3380,65 @@ static void py_emit_dunder_call(PyLSPContext *ctx, const CBMType *recv, const ch } } +/* A class comprehension evaluates its leftmost iterable in the class frame, + * but the implicit body scope skips class locals. Nested comprehensions keep + * the surrounding comprehension frame. Keep class execution marked active so + * legacy bare-name dispatch cannot bypass either frame's lexical bindings. */ +static void py_resolve_class_comprehension(PyLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; + PyKids kids = py_kids(ctx->arena, node); + uint32_t first_for = kids.n; + TSNode first_right = {0}; + for (uint32_t i = 0; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "for_in_clause") == 0) { + first_for = i; + first_right = ts_node_child_by_field_name(child, "right", 5); + break; + } + } + if (first_for == kids.n) + return; + const CBMType *first_type = py_eval_expr_type(ctx, first_right); + py_resolve_calls_in(ctx, first_right); + if (ctx->eval_failure) + return; + const CBMType *first_elem = py_iterable_element_type(ctx, first_type); + CBMScope *saved_scope = ctx->current_scope; + bool saved_comprehension = ctx->class_body_in_comprehension; + CBMScope *parent = saved_comprehension ? saved_scope : ctx->class_body_outer_scope; + CBMScope *frame = py_class_frame_push(ctx, parent); + if (!frame) + return; + py_scope_restore(ctx, frame); + ctx->class_body_in_comprehension = true; + /* Clauses precede the body at evaluation time, although tree-sitter stores + * the body first. Walk each iterable/filter only once, before later binds. */ + for (uint32_t i = first_for; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "for_in_clause") == 0) { + TSNode left = ts_node_child_by_field_name(child, "left", 4); + const CBMType *elem = first_elem; + if (i != first_for) { + TSNode right = ts_node_child_by_field_name(child, "right", 5); + const CBMType *iter = py_eval_expr_type(ctx, right); + py_resolve_calls_in(ctx, right); + if (ctx->eval_failure) + break; + elem = py_iterable_element_type(ctx, iter); + } + py_bind_for_target(ctx, left, elem); + } else { + py_resolve_calls_in(ctx, child); + } + } + for (uint32_t i = 0; i < first_for && !ctx->eval_failure; i++) + py_resolve_calls_in(ctx, py_kid(&kids, i)); + ctx->class_body_in_comprehension = saved_comprehension; + py_scope_restore(ctx, saved_scope); +} + static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) return; @@ -3551,6 +3641,10 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { // iterable's element type, then walk inner expressions. if (strcmp(k, "list_comprehension") == 0 || strcmp(k, "dictionary_comprehension") == 0 || strcmp(k, "set_comprehension") == 0 || strcmp(k, "generator_expression") == 0) { + if (ctx->class_body_outer_scope) { + py_resolve_class_comprehension(ctx, node); + return; + } CBMScope *saved = ctx->current_scope; ctx->current_scope = py_scope_push_checked(ctx); uint32_t cnc = ts_node_named_child_count(node); @@ -3583,15 +3677,19 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { // Recurse: children. We don't push scope for control-flow blocks // here (Python scoping is function-level apart from comprehension / // lambda / class), but we do for nested function / class / lambda. - if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || - strcmp(k, "lambda") == 0) { + if (strcmp(k, "function_definition") == 0 || strcmp(k, "lambda") == 0) { // These are processed by the top-level pass; skip recursion to // avoid double-walking their bodies. return; } + if (strcmp(k, "class_definition") == 0) { + py_exec_class_definition(ctx, node); + return; + } + PyKids nk = py_kids(ctx->arena, node); - for (uint32_t i = 0; i < nk.n; i++) { + for (uint32_t i = 0; i < nk.n && !ctx->eval_failure; i++) { py_resolve_calls_in(ctx, py_kid(&nk, i)); } } @@ -4212,6 +4310,120 @@ static bool py_is_init_method(PyLSPContext *ctx, TSNode func_node) { return nm && (strcmp(nm, "__init__") == 0 || strcmp(nm, "__post_init__") == 0); } +/* Decorator expressions of a decorated definition, in source order. They + * evaluate in the scope enclosing the definition, before it is bound. A bare + * `@name` is no call and emits nothing; `@name(args)` is one. */ +static void py_resolve_decorators(PyLSPContext *ctx, TSNode decorated) { + if (!ctx || ctx->eval_failure) + return; + PyKids kids = py_kids(ctx->arena, decorated); + for (uint32_t i = 0; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "decorator") == 0) + py_resolve_calls_in(ctx, child); + } +} + +/* Bind the name a class-body `def`/`class` statement defines in the class + * frame, so later class-body statements see it (and never the same-named + * module symbol). Its value is exact only for an undecorated function or a + * registered class whose QN is known; anything else is an UNKNOWN shadow. */ +static void py_bind_class_body_def(PyLSPContext *ctx, TSNode stmt) { + if (ctx->eval_failure) + return; + TSNode def = stmt; + bool decorated = strcmp(ts_node_type(stmt), "decorated_definition") == 0; + if (decorated) { + def = ts_node_child_by_field_name(stmt, "definition", 10); + if (ts_node_is_null(def)) + return; + } + const char *kind = ts_node_type(def); + bool is_class = strcmp(kind, "class_definition") == 0; + if (!is_class && strcmp(kind, "function_definition") != 0) + return; + char *name = py_node_text(ctx, ts_node_child_by_field_name(def, "name", 4)); + if (!name || !name[0]) { + py_disable_callable_value_proof(ctx); + return; + } + const char *qn = ctx->class_body_qn + ? cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->class_body_qn, name) + : NULL; + if (is_class) { + const CBMRegisteredType *type = qn ? cbm_registry_lookup_type(ctx->registry, qn) : NULL; + py_scope_bind(ctx, name, + type ? cbm_type_named(ctx->arena, type->qualified_name) : cbm_type_unknown()); + return; + } + const CBMRegisteredFunc *func = qn ? cbm_registry_lookup_func(ctx->registry, qn) : NULL; + if (!decorated && py_func_is_exact_callable_value(func)) { + py_scope_bind_callable(ctx, name, cbm_type_unknown(), func->qualified_name); + } else { + py_scope_bind(ctx, name, cbm_type_unknown()); + } +} + +/* Execute a class definition where it stands: its bases in the enclosing + * scope, then its body statements in source order in a class frame. Calls + * keep the caller the extraction gives them: the enclosing function, or the + * module QN at module level (enclosing_func_qn is left unchanged). + * + * Scoping follows Python: a name bound earlier in the body is visible to + * later body statements (and to decorators of methods), but the class frame + * is not an enclosing scope for anything nested in it. Method bodies are + * resolved later by py_process_class against the module scope, and a nested + * class body chains to the nearest enclosing non-class scope. Each body + * statement is walked exactly once. */ +static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node) { + if (!ctx || ctx->eval_failure || ts_node_is_null(class_node)) + return; + TSNode supers = ts_node_child_by_field_name(class_node, "superclasses", 12); + if (!ts_node_is_null(supers)) + py_resolve_calls_in(ctx, supers); + TSNode body = ts_node_child_by_field_name(class_node, "body", 4); + if (ctx->eval_failure || ts_node_is_null(body)) + return; + + char *cname = py_node_text(ctx, ts_node_child_by_field_name(class_node, "name", 4)); + const char *class_qn = NULL; + if (cname && cname[0]) { + if (ctx->class_body_outer_scope) { + class_qn = ctx->class_body_qn + ? cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->class_body_qn, cname) + : NULL; + } else if (ctx->module_qn && ctx->enclosing_func_qn && ctx->file_module_qn && + strcmp(ctx->enclosing_func_qn, ctx->file_module_qn) == 0) { + class_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, cname); + } + } + + CBMScope *saved_scope = ctx->current_scope; + CBMScope *saved_outer = ctx->class_body_outer_scope; + const char *saved_qn = ctx->class_body_qn; + bool saved_comprehension = ctx->class_body_in_comprehension; + CBMScope *outer = saved_outer ? saved_outer : saved_scope; + CBMScope *frame = py_class_frame_push(ctx, outer); + if (!frame) + return; + py_scope_restore(ctx, frame); + ctx->class_body_outer_scope = outer; + ctx->class_body_qn = class_qn; + ctx->class_body_in_comprehension = false; + + PyKids bk = py_kids(ctx->arena, body); + for (uint32_t i = 0; i < bk.n && !ctx->eval_failure; i++) { + TSNode stmt = py_kid(&bk, i); + py_resolve_calls_in(ctx, stmt); + py_bind_class_body_def(ctx, stmt); + } + + ctx->class_body_outer_scope = saved_outer; + ctx->class_body_qn = saved_qn; + ctx->class_body_in_comprehension = saved_comprehension; + py_scope_restore(ctx, saved_scope); +} + static void py_process_class(PyLSPContext *ctx, TSNode class_node) { if (!ctx || ctx->eval_failure) return; @@ -4874,15 +5086,21 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } else if (strcmp(ck, "function_definition") == 0) { py_bind_module_function(ctx, c); } else if (strcmp(ck, "class_definition") == 0) { - py_bind_module_class(ctx, c); + py_exec_class_definition(ctx, c); + if (!ctx->eval_failure) + py_bind_module_class(ctx, c); } else if (strcmp(ck, "decorated_definition") == 0) { + py_resolve_decorators(ctx, c); TSNode def = ts_node_child_by_field_name(c, "definition", 10); - if (!ts_node_is_null(def)) { + if (!ctx->eval_failure && !ts_node_is_null(def)) { const char *dk = ts_node_type(def); - if (strcmp(dk, "function_definition") == 0) + if (strcmp(dk, "function_definition") == 0) { py_bind_module_function(ctx, def); - else if (strcmp(dk, "class_definition") == 0) - py_bind_module_class(ctx, def); + } else if (strcmp(dk, "class_definition") == 0) { + py_exec_class_definition(ctx, def); + if (!ctx->eval_failure) + py_bind_module_class(ctx, def); + } } } else if (strcmp(ck, "expression_statement") == 0) { /* An expression statement is resolved for calls, not scanned as a diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index d733b31538..b20e733279 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -61,8 +61,13 @@ typedef struct { // caller QNs. const char *enclosing_func_qn; const char *enclosing_class_qn; - const char *module_qn; // Python symbol scope (package __init__ omitted) - const char *file_module_qn; // raw file identity; borrowed for this API call + // Active class execution: nested class bodies skip this class frame. + // Local classes have no extracted graph QN, so class_body_qn stays NULL. + CBMScope *class_body_outer_scope; + const char *class_body_qn; + bool class_body_in_comprehension; // retain enclosing comprehension bindings + const char *module_qn; // Python symbol scope (package __init__ omitted) + const char *file_module_qn; // raw file identity; borrowed for this API call // Output: resolved calls accumulate here. CBMResolvedCallArray *resolved_calls; @@ -174,6 +179,8 @@ void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); void cbm_py_lsp_test_memo_fail_after(int successful_allocations); void cbm_py_lsp_test_depth_fail(bool enabled); void cbm_py_lsp_test_scope_fail_after(int successful_allocations); +// Separate from the package symbol-scope seam; only new class execution frames. +void cbm_py_lsp_test_class_frame_fail_after(int successful_allocations); bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, int def_count); const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a7080137b9..633e47aacf 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8530,6 +8530,153 @@ TEST(pipeline_python_module_callers_parallel) { return python_module_callers_pipeline_case(true); } +/* Distinct targets keep each newly walked class/decorator site observable + * even though the stored graph aggregates calls by caller/target pair. */ +static int python_class_body_pipeline_case(bool parallel) { + static const char helpers[] = + "def top_field():\n return 1\n" + "def nested_field():\n return 2\n" + "def class_decorator(value):\n return lambda target: target\n" + "def class_argument():\n return 3\n" + "def method_decorator(value):\n return lambda target: target\n" + "def method_argument():\n return 4\n" + "def method_control():\n return 5\n" + "def local_field():\n return 6\n" + "def method_local_field():\n return 7\n" + "def base_factory():\n return object\n" + "def package_field():\n return 8\n" + "def package_decorator(value):\n return lambda target: target\n" + "def package_argument():\n return 9\n"; + static const char app[] = + "from helpers import top_field, nested_field, class_decorator, class_argument\n" + "from helpers import method_decorator, method_argument, method_control\n" + "from helpers import local_field, method_local_field, base_factory\n" + "class Base(base_factory()):\n" + " value = top_field()\n" + " @method_decorator(method_argument())\n" + " def run(self):\n" + " method_control()\n" + " class MethodLocal:\n" + " value = method_local_field()\n" + "@class_decorator(class_argument())\n" + "class Admin:\n" + " class Meta:\n" + " value = nested_field()\n" + "def outer():\n" + " class Local:\n" + " value = local_field()\n" + " return Local\n"; + static const char package[] = + "from helpers import package_field, package_decorator, package_argument\n" + "@package_decorator(package_argument())\n" + "class Package:\n" + " value = package_field()\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + } expected[] = { + {"app", "Module", "helpers.base_factory"}, + {"app", "Module", "helpers.top_field"}, + {"app", "Module", "helpers.nested_field"}, + {"app", "Module", "helpers.class_decorator"}, + {"app", "Module", "helpers.class_argument"}, + {"app", "Module", "helpers.method_decorator"}, + {"app", "Module", "helpers.method_argument"}, + {"app.Base.run", "Method", "helpers.method_control"}, + {"app.Base.run", "Method", "helpers.method_local_field"}, + {"app.outer", "Function", "helpers.local_field"}, + {"pkg.__init__", "Module", "helpers.package_decorator"}, + {"pkg.__init__", "Module", "helpers.package_argument"}, + {"pkg.__init__", "Module", "helpers.package_field"}, + }; + char root[512], path[768]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-class-body-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python class-body fixture"); + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file(root, "helpers.py", helpers) && + python_module_write_fixture_file(root, "app.py", app) && + python_module_write_fixture_file(root, "pkg/__init__.py", package); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "class_body_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "# inert parallel routing file\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python class-body fixture"); + } + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {0}; + bool saved_ok = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + free(saved[0]); + free(saved[1]); + th_rmtree(root); + FAIL("failed to save Python class-body fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + n = snprintf(path, sizeof(path), "%s/class-body.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + int file_count = 0; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (parallel ? 55 : 3); + cbm_store_free_nodes(files, file_count); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + bool edge = project && python_module_exact_lsp_edge(store, project, expected[i].caller, + expected[i].caller_label, + expected[i].target, "Function"); + correct = correct && edge; + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, 0); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pipeline_python_class_body_calls_sequential) { + return python_class_body_pipeline_case(false); +} +TEST(pipeline_python_class_body_calls_parallel) { + return python_class_body_pipeline_case(true); +} + /* Python file identity keeps .__init__; its exported symbols do not. */ static bool python_init_local_inheritance(cbm_store_t *store, const char *project, const char *module) { @@ -17122,6 +17269,8 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { SUITE(pipeline) { RUN_TEST(pipeline_python_module_callers_sequential); RUN_TEST(pipeline_python_module_callers_parallel); + RUN_TEST(pipeline_python_class_body_calls_sequential); + RUN_TEST(pipeline_python_class_body_calls_parallel); RUN_TEST(pipeline_python_init_scope_sequential); RUN_TEST(pipeline_python_init_scope_parallel); RUN_TEST(pipeline_python_init_raw_scope_failure_sequential); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index b16f5f87a8..8ffe4540d9 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -3302,7 +3302,482 @@ TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback) { PASS(); } +/* Class execution keeps the extracted caller and exact source occurrence. */ +static bool pylsp_class_site_unjoined(const CBMFileResult *r, const char *source, + const char *site_text, const char *caller) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source); + uint32_t end = start + (uint32_t)strlen(site_text); + const CBMCall *call = NULL; + int count = 0; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *candidate = &r->calls.items[i]; + if (candidate->enclosing_func_qn && strcmp(candidate->enclosing_func_qn, caller) == 0 && + candidate->site_start_byte == start && candidate->site_end_byte == end) { + call = candidate; + count++; + } + } + if (count != 1 || cbm_pipeline_find_lsp_resolution(&r->resolved_calls, call, false)) + return false; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *hit = &r->resolved_calls.items[i]; + if (hit->kind == CBM_RESOLVED_INVOCATION && hit->site_start_byte == start && + hit->site_end_byte == end) + return false; + } + return true; +} + +typedef struct { + const char *site; + const char *caller; + const char *target; /* NULL requires an extracted, unresolved occurrence. */ +} PylspClassSite; + +static bool pylsp_class_sites(const CBMFileResult *r, const char *source, + const PylspClassSite *sites, size_t count) { + if (!r || r->has_error || r->parse_incomplete || r->lsp_skipped || !r->module_qn) + return false; + for (size_t i = 0; i < count; i++) { + const PylspClassSite *s = &sites[i]; + if (strcmp(s->caller, r->module_qn) != 0 && !pylsp_module_has_def(r, s->caller)) + return false; + if (s->target) { + if (!pylsp_module_has_def(r, s->target) || + !pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, s->site, s->caller, + s->target)) + return false; + } else if (!pylsp_class_site_unjoined(r, source, s->site, s->caller)) { + return false; + } + } + return true; +} + +TEST(pylsp_classbody_calls_join_extracted_caller) { + static const char source[] = + "def field(n):\n return n\n" + "class Svc:\n" + " @staticmethod\n def make(n):\n return Svc()\n" + "class Model:\n" + " first = field(11)\n second = Svc(12)\n third = Svc.make(13)\n" + "def control():\n field(21)\n Svc(22)\n Svc.make(23)\n"; + static const char *const paths[] = {"main.py", "pkg/__init__.py"}; + static const char *const modules[] = {"test.main", "test.pkg.__init__"}; + static const char *const scopes[] = {"test.main", "test.pkg"}; + bool correct = true; + for (int path = 0; path < 2; path++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[path], 0, NULL, NULL); + char field[64], svc[64], make[64], control[64]; + snprintf(field, sizeof(field), "%s.field", scopes[path]); + snprintf(svc, sizeof(svc), "%s.Svc", scopes[path]); + snprintf(make, sizeof(make), "%s.Svc.make", scopes[path]); + snprintf(control, sizeof(control), "%s.control", scopes[path]); + const PylspClassSite sites[] = {{"field(11)", modules[path], field}, + {"Svc(12)", modules[path], svc}, + {"Svc.make(13)", modules[path], make}, + {"field(21)", control, field}, + {"Svc(22)", control, svc}, + {"Svc.make(23)", control, make}}; + correct = pylsp_class_sites(r, source, sites, 6) && correct; + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_classbody_names_visible_to_later_statements_only) { + static const char source[] = + "def helper(n):\n return n\n" + "class Svc:\n def run(self, n):\n return n\n" + "class Model:\n" + " obj = Svc()\n result = obj.run(101)\n" + " def helper(self):\n return 2\n" + " later = helper(None)\n" + " def m(self):\n return helper(102)\n" + " def n(self):\n return obj.run(103)\n" + "class Shadow:\n helper = None\n value = helper(104)\n" + "def control():\n obj = Svc()\n obj.run(105)\n helper(106)\n"; + const PylspClassSite sites[] = { + {"obj.run(101)", "test.main", "test.main.Svc.run"}, + {"helper(None)", "test.main", "test.main.Model.helper"}, + {"helper(102)", "test.main.Model.m", "test.main.helper"}, + {"obj.run(103)", "test.main.Model.n", NULL}, + {"helper(104)", "test.main", NULL}, + {"obj.run(105)", "test.main.control", "test.main.Svc.run"}, + {"helper(106)", "test.main.control", "test.main.helper"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 7); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_decorator_and_base_calls_join_extracted_caller) { + static const char source[] = "def deco(fn):\n return fn\n" + "def deco_arg(n):\n return deco\n" + "def base_factory(n):\n return object\n" + "class Svc:\n def run(self):\n return 1\n" + "@deco_arg(11)\ndef top():\n pass\n" + "@deco\ndef bare():\n pass\n" + "@deco_arg(12)\nclass Model(base_factory(13)):\n" + " marker = Svc()\n" + " @deco_arg(marker.run())\n def m(self):\n pass\n" + " @property\n def p(self):\n return 1\n" + "def outer():\n @deco_arg(14)\n def inner():\n pass\n" + " return inner\n"; + bool correct = true; + for (int package = 0; package < 2; package++) { + const char *scope = package ? "test.pkg" : "test.main"; + const char *module = package ? "test.pkg.__init__" : "test.main"; + char deco[64], base[64], run[64], outer[64]; + snprintf(deco, sizeof(deco), "%s.deco_arg", scope); + snprintf(base, sizeof(base), "%s.base_factory", scope); + snprintf(run, sizeof(run), "%s.Svc.run", scope); + snprintf(outer, sizeof(outer), "%s.outer", scope); + const PylspClassSite sites[] = { + {"deco_arg(11)", module, deco}, {"deco_arg(12)", module, deco}, + {"base_factory(13)", module, base}, {"deco_arg(marker.run())", module, deco}, + {"marker.run()", module, run}, {"deco_arg(14)", outer, deco}}; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + package ? "pkg/__init__.py" : "main.py", 0, NULL, NULL); + correct = pylsp_class_sites(r, source, sites, 6) && correct; + if (r) { + const char *bare_sites[] = {"@deco\n", "@property\n"}; + for (int b = 0; b < 2; b++) { + const char *hit = strstr(source, bare_sites[b]); + uint32_t lo = (uint32_t)(hit - source); + uint32_t hi = lo + (uint32_t)strlen(bare_sites[b]); + for (int i = 0; i < r->calls.count; i++) + correct = !(r->calls.items[i].site_start_byte >= lo && + r->calls.items[i].site_start_byte < hi) && + correct; + for (int i = 0; i < r->resolved_calls.count; i++) + correct = !(r->resolved_calls.items[i].kind == CBM_RESOLVED_INVOCATION && + r->resolved_calls.items[i].site_start_byte >= lo && + r->resolved_calls.items[i].site_start_byte < hi) && + correct; + } + cbm_free_result(r); + } + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_nested_and_local_classbody_calls_join_extracted_caller) { + static const char source[] = + "def field(n):\n return n\n" + "class Svc:\n def run(self):\n return 1\n" + "class Outer:\n tag = Svc()\n" + " class Meta:\n ordering = field(31)\n probe = tag.run()\n" + " inner = Meta()\n" + "def build():\n before = field(32)\n" + " class Local:\n value = field(33)\n" + " class Inner:\n value = field(34)\n" + " return Local\n" + "class Controller:\n def build(self):\n" + " class Local:\n value = field(35)\n" + " return Local\n"; + const PylspClassSite sites[] = { + {"field(31)", "test.main", "test.main.field"}, + {"tag.run()", "test.main", NULL}, + {"Meta()", "test.main", "test.main.Outer.Meta"}, + {"field(32)", "test.main.build", "test.main.field"}, + {"field(33)", "test.main.build", "test.main.field"}, + {"field(34)", "test.main.build", "test.main.field"}, + {"field(35)", "test.main.Controller.build", "test.main.field"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 7) && + !pylsp_module_has_def(r, "test.main.build.Local") && + !pylsp_module_has_def(r, "test.main.Controller.build.Local"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_comprehensions_separate_iterable_and_body_scopes) { + static const char source[] = + "class ModuleSvc:\n def run(self, n):\n return n\n" + "class ClassSvc:\n" + " def run(self, n):\n return n\n" + " def items(self) -> list[ModuleSvc]:\n return []\n" + "shared = ModuleSvc()\n" + "class Model:\n" + " shared = ClassSvc()\n local = ClassSvc()\n" + " a = [item.run(41) for item in local.items()]\n" + " b = {shared.run(42) for item in [1]}\n" + " c = {item: shared.run(43) for item in [1]}\n" + " d = (shared.run(44) for item in [1])\n" + " e = [local.run(45) for item in [1]]\n" + " f = [item for item in [1] if local.run(46)]\n" + " g = [item for item in [1] for other in local.items(47)]\n" + " h = [[inner.run(48) for inner in [outer]] for outer in [ModuleSvc()]]\n" + " class Nested:\n" + " shared = ClassSvc()\n" + " a = [shared.run(49) for item in [1]]\n" + " after = shared.run(50)\n" + "def control():\n return [item.run(51) for item in [ModuleSvc()]]\n"; + const PylspClassSite sites[] = { + {"local.items()", "test.main", "test.main.ClassSvc.items"}, + {"item.run(41)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(42)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(43)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(44)", "test.main", "test.main.ModuleSvc.run"}, + {"local.run(45)", "test.main", NULL}, + {"local.run(46)", "test.main", NULL}, + {"local.items(47)", "test.main", NULL}, + {"inner.run(48)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(49)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(50)", "test.main", "test.main.ClassSvc.run"}, + {"item.run(51)", "test.main.control", "test.main.ModuleSvc.run"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 12) && + pylsp_module_has_def(r, "test.main.ClassSvc.run") && + pylsp_module_has_def(r, "test.main.ClassSvc.items"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_legacy_dispatch_cannot_bypass_or_publish_shadows) { + static const char source[] = "def wanted(n):\n return n\n" + "def other(n):\n return n\n" + "class Svc:\n def run(self, n):\n return n\n" + "table = {'x': wanted}\n" + "fn = lambda obj: obj.run(61)\n" + "stable = lambda obj: obj.run(62)\n" + "class Shadow:\n" + " table = None\n a = table['x'](63)\n" + " fn = None\n b = fn(Svc(64))\n" + "class Replacements:\n" + " table = {'x': other}\n c = table['x'](65)\n" + " stable = lambda obj: obj.run(66)\n" + "outside = table['x'](67)\n" + "outside_lambda = stable(Svc(68))\n"; + const PylspClassSite sites[] = { + {"table['x'](63)", "test.main", NULL}, + {"Svc(64)", "test.main", "test.main.Svc"}, + {"table['x'](65)", "test.main", NULL}, + {"table['x'](67)", "test.main", "test.main.wanted"}, + {"Svc(68)", "test.main", "test.main.Svc"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 5) && + pylsp_module_has_def(r, "test.main.other") && + pylsp_module_has_def(r, "test.main.Svc.run"); + /* A replay has the lambda body's site, not the invocation's site. Verify + * those exact sites independently so output dedup cannot hide a replay. */ + if (r) { + const char *body[] = {"obj.run(61)", "obj.run(62)", "obj.run(66)"}; + for (int b = 0; b < 3; b++) { + uint32_t lo = (uint32_t)(strstr(source, body[b]) - source); + uint32_t hi = lo + (uint32_t)strlen(body[b]); + int extracted = 0, resolved = 0; + for (int i = 0; i < r->calls.count; i++) + extracted += r->calls.items[i].site_start_byte == lo && + r->calls.items[i].site_end_byte == hi; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *hit = &r->resolved_calls.items[i]; + if (hit->kind == CBM_RESOLVED_INVOCATION && hit->site_start_byte == lo && + hit->site_end_byte == hi) { + resolved++; + correct = hit->caller_qn && strcmp(hit->caller_qn, "test.main.") == 0 && + hit->callee_qn && strcmp(hit->callee_qn, "test.main.Svc.run") == 0 && + correct; + } + } + correct = extracted == 1 && resolved == (b == 1 ? 1 : 0) && correct; + } + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_frame_failure_restores_context_and_stops) { + static const char source[] = "class Outer:\n leaked = 1\n" + " first = hit(71)\n" + " values = [hit(72) for item in [1]]\n" + "later = hit(73)\n"; + TSParser *parser = ts_parser_new(); + bool setup = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + setup ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool correct = tree && !ts_node_has_error(ts_tree_root_node(tree)); + CBMLSPDef defs[] = {{.qualified_name = "test.main.hit", + .short_name = "hit", + .label = "Function", + .def_module_qn = "test.main", + .lang = CBM_LANG_PYTHON}}; + for (int failure = 0; tree && failure < 2; failure++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool registered = cbm_py_lsp_test_register_defs(&arena, ®, defs, 1); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), ®, "test.main", &out); + CBMScope *root_scope = ctx.current_scope; + ctx.enclosing_func_qn = "saved.caller"; + ctx.enclosing_class_qn = "saved.class"; + cbm_py_lsp_test_class_frame_fail_after(failure); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = registered && root_scope && ctx.eval_failure == CBM_LSP_SCOPE_FAILED && + ctx.current_scope == root_scope && !ctx.class_body_outer_scope && + !ctx.class_body_qn && !ctx.class_body_in_comprehension && + strcmp(ctx.enclosing_func_qn, "saved.caller") == 0 && + strcmp(ctx.enclosing_class_qn, "saved.class") == 0 && out.count == 0 && + !cbm_scope_lookup_local(root_scope, "leaked") && + !cbm_scope_lookup_local(root_scope, "Outer") && + !cbm_scope_lookup_local(root_scope, "later") && correct; + (void)cbm_lsp_work_take(); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + correct = cbm_lsp_work_take() == 0 && out.count == 0 && correct; + cbm_arena_destroy(&arena); + } + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + /* The pre-existing package seam must still ignore ordinary module names, + * including the new class/comprehension frames. */ + static const char healthy[] = "def hit(n):\n return n\n" + "class Outer:\n first = hit(74)\n" + " values = [hit(75) for item in [1]]\n"; + cbm_py_lsp_test_scope_fail_after(0); + CBMFileResult *r = extract_py(healthy); + cbm_py_lsp_test_scope_fail_after(-1); + const PylspClassSite sites[] = {{"hit(74)", "test.main", "test.main.hit"}, + {"hit(75)", "test.main", "test.main.hit"}}; + bool recovered = pylsp_class_sites(r, healthy, sites, 2); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + +TEST(pylsp_class_frame_failure_rolls_back_raw_and_cross_outputs) { + static const char source[] = "class Outer:\n item = S()\n item + item\n item.m()\n" + " class Inner:\n later = S(81)\n" + "after = S(82)\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.main.S", + .short_name = "S", + .label = "Class", + .def_module_qn = "test.main", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.main.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.main.S", + .def_module_qn = "test.main", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.main.S.__add__", + .short_name = "__add__", + .label = "Method", + .receiver_type = "test.main.S", + .def_module_qn = "test.main", + .return_types = "S", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + for (int prebuilt = 0; prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = prebuilt ? cbm_py_build_cross_registry(&shared, defs, 3) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCall prior = { + .caller_qn = "test.main", + .callee_qn = "test.main.S.__add__", + .kind = CBM_RESOLVED_INVOCATION, + .confidence = 0.1f, + .strategy = "prior.strategy", + .reason = "prior.reason", + .site_start_byte = (uint32_t)(strstr(source, "item + item") - source), + .site_end_byte = (uint32_t)(strstr(source, "item + item") - source) + 11}; + CBMResolvedCall saved_prior; + memcpy(&saved_prior, &prior, sizeof(prior)); + CBMCall prior_synthetic = {.callee_name = "completed.synthetic"}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + CBMCallArray synthetic = {.items = &prior_synthetic, .count = 1, .cap = 1}; + cbm_py_lsp_test_class_frame_fail_after(1); + CBMLSPStatus status = + prebuilt + ? cbm_run_py_lsp_cross_with_registry_status(&arena, source, (int)strlen(source), + "test.main", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), "test.main", + defs, 3, NULL, NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = (!prebuilt || base) && status == CBM_LSP_SCOPE_FAILED && out.count == 1 && + synthetic.count == 1 && memcmp(&prior, &saved_prior, sizeof(prior)) == 0 && + memcmp(&out.items[0], &saved_prior, sizeof(prior)) == 0 && + strcmp(prior_synthetic.callee_name, "completed.synthetic") == 0 && correct; + bool complete = + prebuilt ? cbm_run_py_lsp_cross_with_registry(&arena, source, (int)strlen(source), + "test.main", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.main", defs, + 3, NULL, NULL, 0, NULL, &out, &synthetic); + correct = complete && out.count > 1 && synthetic.count > 1 && + out.items[0].confidence > saved_prior.confidence && correct; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + static const char raw_source[] = "class S:\n def m(self):\n return 1\n" + "class Outer:\n item = S()\n item.m()\n" + " class Inner:\n later = S(83)\n"; + cbm_py_lsp_test_class_frame_fail_after(2); + CBMFileResult *r = extract_py(raw_source); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = r && r->has_error && r->lsp_skipped && !r->parse_incomplete && r->error_msg && + strcmp(r->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && r->resolved_calls.count == 0 && + r->calls.count > 0 && correct; + if (r) { + cbm_result_compact(r); + correct = r->error_msg && strcmp(r->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && correct; + cbm_free_result(r); + } + r = extract_py(raw_source); + const PylspClassSite sites[] = {{"item.m()", "test.main", "test.main.S.m"}, + {"S(83)", "test.main", "test.main.S"}}; + bool recovered = pylsp_class_sites(r, raw_source, sites, 2); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_classbody_calls_join_extracted_caller); + RUN_TEST(pylsp_classbody_names_visible_to_later_statements_only); + RUN_TEST(pylsp_decorator_and_base_calls_join_extracted_caller); + RUN_TEST(pylsp_nested_and_local_classbody_calls_join_extracted_caller); + RUN_TEST(pylsp_class_comprehensions_separate_iterable_and_body_scopes); + RUN_TEST(pylsp_class_legacy_dispatch_cannot_bypass_or_publish_shadows); + RUN_TEST(pylsp_class_frame_failure_restores_context_and_stops); + RUN_TEST(pylsp_class_frame_failure_rolls_back_raw_and_cross_outputs); + RUN_TEST(pylsp_init_scope_context_and_helper); RUN_TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations); RUN_TEST(pylsp_init_scope_cross_signatures_keep_definition_language); From 1ebacc36f82005bc64cfc8fc3ee610215d4e2f21 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 22:18:56 +0200 Subject: [PATCH 7/9] fix(python): preserve ordered namespace metadata Capture ordered Python namespace facts and retain them through compact, spill, and versioned LSP surfaces. Reject unavailable metadata before incremental reuse and rebuild on Python namespace changes or deletion. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 218 +++++++++++ internal/cbm/cbm.h | 83 ++++ internal/cbm/extract_imports.c | 486 +++++++++++++++++++++++ internal/cbm/result_compact.c | 62 ++- internal/cbm/result_spill.c | 7 +- internal/cbm/result_spill.h | 5 +- src/pipeline/lsp_surface.c | 310 ++++++++++++++- src/pipeline/lsp_surface.h | 34 +- src/pipeline/pipeline_incremental.c | 25 +- src/store/store.h | 11 +- tests/test_extraction.c | 489 +++++++++++++++++++++++ tests/test_pipeline.c | 583 +++++++++++++++++++++++++++- 12 files changed, 2290 insertions(+), 23 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 463c44074d..90b71d8c2b 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -110,6 +110,190 @@ void cbm_reset_profile(void) { atomic_store(&total_files, 0); } +/* New facts use checked allocation and atomic publication; legacy extraction + * arrays retain their existing contracts. */ +static bool py_namespace_nonempty(const char *s) { + return s && s[0]; +} + +static bool py_namespace_fact_valid(const CBMPyNamespaceFact *f) { + if (!f || f->name_count < 0 || (size_t)f->name_count > SIZE_MAX / sizeof(*f->names) || + (f->name_count == 0 ? f->names != NULL : f->names == NULL)) + return false; + bool literal = f->kind == CBM_PY_NS_ALL_SET || f->kind == CBM_PY_NS_ALL_APPEND; + if (!literal && (f->names || f->name_count || f->sequence_kind != CBM_PY_NS_SEQUENCE_NONE)) + return false; + if (f->kind != CBM_PY_NS_OWN_DEF && f->def_kind != CBM_PY_NS_DEF_NONE) + return false; + if (f->kind != CBM_PY_NS_IMPORT_MODULE && f->flags) + return false; + if (f->kind != CBM_PY_NS_IMPORT_NAME && f->kind != CBM_PY_NS_IMPORT_STAR && f->relative_level) + return false; + bool unknown = f->kind == CBM_PY_NS_SHADOW || f->kind == CBM_PY_NS_UNKNOWN || + f->kind == CBM_PY_NS_ALL_UNKNOWN; + if (!unknown && f->reason != CBM_PY_NS_REASON_NONE) + return false; + switch (f->kind) { + case CBM_PY_NS_OWN_DEF: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->def_kind == CBM_PY_NS_DEF_FUNCTION || f->def_kind == CBM_PY_NS_DEF_CLASS); + case CBM_PY_NS_IMPORT_MODULE: + return py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->module_name) && + !f->member_name && !(f->flags & ~CBM_PY_NS_IMPORT_BINDS_ROOT); + case CBM_PY_NS_IMPORT_NAME: + case CBM_PY_NS_IMPORT_STAR: + if (!f->module_name || (!f->module_name[0] && !f->relative_level)) + return false; + return f->kind == CBM_PY_NS_IMPORT_NAME + ? py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->member_name) + : !f->local_name && !f->member_name; + case CBM_PY_NS_SHADOW: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED); + case CBM_PY_NS_DELETE: + return py_namespace_nonempty(f->local_name) && strcmp(f->local_name, "__all__") != 0 && + !f->module_name && !f->member_name; + case CBM_PY_NS_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_COMPOUND || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_SET: + case CBM_PY_NS_ALL_APPEND: + if (f->local_name || f->module_name || f->member_name || + (f->sequence_kind != CBM_PY_NS_SEQUENCE_LIST && + f->sequence_kind != CBM_PY_NS_SEQUENCE_TUPLE)) + return false; + for (int i = 0; i < f->name_count; i++) { + if (!f->names[i]) + return false; + } + return true; + case CBM_PY_NS_ALL_DELETE: + return !f->local_name && !f->module_name && !f->member_name; + default: + return false; + } +} + +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *f) { + if (!f) + return false; + bool empty = !f->items && f->count == 0 && f->cap == 0; + if (f->version == 0) + return f->status == CBM_PY_NS_NOT_CAPTURED && f->failure == CBM_PY_NS_FAILURE_NONE && empty; + if (f->version != CBM_PY_NAMESPACE_FACTS_VERSION || f->language < 0 || + f->language >= CBM_LANG_COUNT) + return false; + switch (f->status) { + case CBM_PY_NS_NOT_CAPTURED: + return empty && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_NOT_APPLICABLE: + return empty && f->language != CBM_LANG_PYTHON && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_INCOMPLETE: + return empty && f->failure >= CBM_PY_NS_FAILURE_INVALID_INPUT && + f->failure <= CBM_PY_NS_FAILURE_LIMIT; + case CBM_PY_NS_COMPLETE: + if (f->language != CBM_LANG_PYTHON || f->failure != CBM_PY_NS_FAILURE_NONE || + f->count < 0 || f->cap < f->count || (size_t)f->cap > SIZE_MAX / sizeof(*f->items) || + (f->count == 0 ? !empty : f->items == NULL)) + return false; + for (int i = 0; i < f->count; i++) { + if (!py_namespace_fact_valid(&f->items[i])) + return false; + } + return true; + default: + return false; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_copy_left = -1; +void cbm_py_namespace_test_copy_fail_after(int successful_allocations) { + py_namespace_copy_left = successful_allocations; +} +#endif + +static void *py_namespace_copy_alloc(CBMArena *arena, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_copy_left == 0) + return NULL; + if (py_namespace_copy_left > 0) + py_namespace_copy_left--; +#endif + return cbm_arena_alloc(arena, size); +} + +static bool py_namespace_copy_string(CBMArena *arena, const char *source, const char **out) { + *out = NULL; + if (!source) + return true; + size_t len = strlen(source); + if (len == SIZE_MAX) + return false; + char *copy = py_namespace_copy_alloc(arena, len + 1); + if (!copy) + return false; + memcpy(copy, source, len + 1); + *out = copy; + return true; +} + +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out) { + if (!out || source == out) + return false; + CBMPyNamespaceFacts failure = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_INVALID_INPUT}; + if (!arena || !cbm_py_namespace_facts_valid(source)) { + *out = failure; + return false; + } + failure.language = source->language; + failure.failure = CBM_PY_NS_FAILURE_ALLOCATION; + CBMPyNamespaceFacts copy = *source; + copy.items = NULL; + copy.cap = copy.count; + if (copy.count) { + copy.items = py_namespace_copy_alloc(arena, (size_t)copy.count * sizeof(*copy.items)); + if (!copy.items) + goto failed; + memcpy(copy.items, source->items, (size_t)copy.count * sizeof(*copy.items)); + } + for (int i = 0; i < copy.count; i++) { + const CBMPyNamespaceFact *src = &source->items[i]; + CBMPyNamespaceFact *dst = ©.items[i]; + if (!py_namespace_copy_string(arena, src->local_name, &dst->local_name) || + !py_namespace_copy_string(arena, src->module_name, &dst->module_name) || + !py_namespace_copy_string(arena, src->member_name, &dst->member_name)) + goto failed; + dst->names = NULL; + if (src->name_count) { + dst->names = + py_namespace_copy_alloc(arena, (size_t)src->name_count * sizeof(*dst->names)); + if (!dst->names) + goto failed; + for (int j = 0; j < src->name_count; j++) { + if (!py_namespace_copy_string(arena, src->names[j], &dst->names[j])) + goto failed; + } + } + } + *out = copy; + return true; +failed: + *out = failure; + return false; +} + // --- Growable array push functions --- #define GROW_ARRAY(arr, arena) \ @@ -2337,6 +2521,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C return NULL; } + result->py_namespace.version = CBM_PY_NAMESPACE_FACTS_VERSION; + result->py_namespace.language = language; + result->py_namespace.status = + language == CBM_LANG_PYTHON ? CBM_PY_NS_NOT_CAPTURED : CBM_PY_NS_NOT_APPLICABLE; cbm_work_arena_take(&result->arena); CBMArena *a = &result->arena; @@ -2523,6 +2711,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // then a single unified cursor walk handles the remaining 7 extractors. cbm_extract_definitions(&ctx); cbm_extract_imports(&ctx); + cbm_extract_python_namespace_facts(&ctx); cbm_extract_unified(&ctx); result->tree_nodes = ts_node_descendant_count(root); result->walk_nodes_visited = ctx.walk_nodes_visited; @@ -3229,6 +3418,35 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua CBMFileResult *result = extract_file_ex_body(source, source_len, language, project, rel_path, timeout_micros, extra_defines, include_paths, macro_table, return_type_table, &scratch); + /* Even early parse/quarantine exits publish conservative Python metadata. + * No tree is reparsed, and existing best-effort extraction flags stay intact. */ + if (result && language == CBM_LANG_PYTHON && + result->py_namespace.status == CBM_PY_NS_NOT_CAPTURED) { + CBMPyNamespaceFailure failure = CBM_PY_NS_FAILURE_NONE; + if (!result->module_qn || !result->module_qn[0]) { + if (!project || !project[0] || !rel_path || !rel_path[0]) { + failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else { + result->module_qn = + cbm_fqn_module_source_lang(&result->arena, project, rel_path, language); + if (!result->module_qn || !result->module_qn[0]) + failure = CBM_PY_NS_FAILURE_ALLOCATION; + } + } + if (failure) { + result->py_namespace = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = failure}; + } else { + CBMExtractCtx facts_ctx = {.arena = &result->arena, + .result = result, + .source = source, + .source_len = source_len, + .language = language}; + cbm_extract_python_namespace_facts(&facts_ctx); + } + } /* !tl_scratch_live: a nested extraction (an embedded language inside this * file) may already have parked its own; never overwrite it. */ /* Kept up to CBM_EXTRACT_SCRATCH_KEEP_BYTES, grown blocks included: the diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 56f57dcd4a..2a83d0bd79 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -310,6 +310,79 @@ typedef struct { bool is_default; // ES default import (`import X from "Y"`), JS/TS only (#1916) } CBMImport; +/* Content-only ordered Python namespace facts; no resolved target identities. */ +#define CBM_PY_NAMESPACE_FACTS_VERSION 1u + +typedef enum { + CBM_PY_NS_NOT_CAPTURED = 0, + CBM_PY_NS_NOT_APPLICABLE = 1, + CBM_PY_NS_COMPLETE = 2, + CBM_PY_NS_INCOMPLETE = 3 +} CBMPyNamespaceStatus; +typedef enum { + CBM_PY_NS_FAILURE_NONE = 0, + CBM_PY_NS_FAILURE_INVALID_INPUT = 1, + CBM_PY_NS_FAILURE_ALLOCATION = 2, + CBM_PY_NS_FAILURE_LIMIT = 3 +} CBMPyNamespaceFailure; +typedef enum { + CBM_PY_NS_OWN_DEF = 1, + CBM_PY_NS_IMPORT_MODULE = 2, + CBM_PY_NS_IMPORT_NAME = 3, + CBM_PY_NS_IMPORT_STAR = 4, + CBM_PY_NS_SHADOW = 5, + CBM_PY_NS_DELETE = 6, + CBM_PY_NS_UNKNOWN = 7, + CBM_PY_NS_ALL_SET = 8, + CBM_PY_NS_ALL_APPEND = 9, + CBM_PY_NS_ALL_UNKNOWN = 10, + CBM_PY_NS_ALL_DELETE = 11 +} CBMPyNamespaceFactKind; +typedef enum { + CBM_PY_NS_DEF_NONE = 0, + CBM_PY_NS_DEF_FUNCTION = 1, + CBM_PY_NS_DEF_CLASS = 2 +} CBMPyNamespaceDefKind; +typedef enum { + CBM_PY_NS_SEQUENCE_NONE = 0, + CBM_PY_NS_SEQUENCE_LIST = 1, + CBM_PY_NS_SEQUENCE_TUPLE = 2 +} CBMPyNamespaceSequenceKind; +typedef enum { + CBM_PY_NS_REASON_NONE = 0, + CBM_PY_NS_REASON_VALUE = 1, + CBM_PY_NS_REASON_DECORATED = 2, + CBM_PY_NS_REASON_UNPROVEN_CLASS = 3, + CBM_PY_NS_REASON_COMPOUND = 4, + CBM_PY_NS_REASON_EFFECT = 5, + CBM_PY_NS_REASON_UNSUPPORTED = 6, + CBM_PY_NS_REASON_PARSE = 7 +} CBMPyNamespaceUnknownReason; +enum { CBM_PY_NS_IMPORT_BINDS_ROOT = 1u << 0 }; + +typedef struct { + CBMPyNamespaceFactKind kind; + CBMPyNamespaceDefKind def_kind; + CBMPyNamespaceSequenceKind sequence_kind; + CBMPyNamespaceUnknownReason reason; + uint32_t flags; + uint32_t relative_level; + const char *local_name; + const char *module_name; + const char *member_name; + const char **names; + int name_count; +} CBMPyNamespaceFact; +typedef struct { + uint32_t version; + CBMLanguage language; + CBMPyNamespaceStatus status; + CBMPyNamespaceFailure failure; + CBMPyNamespaceFact *items; + int count; + int cap; +} CBMPyNamespaceFacts; + typedef enum { CBM_USAGE_VALUE = 0, CBM_USAGE_CALL_REFERENCE, @@ -551,6 +624,7 @@ typedef struct CBMFileResult { CBMDefArray defs; CBMCallArray calls; CBMImportArray imports; + CBMPyNamespaceFacts py_namespace; CBMUsageArray usages; CBMThrowArray throws; CBMRWArray rw; @@ -909,6 +983,15 @@ int cbm_macro_extraction_enabled(void); // --- Internal helpers used by extractors --- +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx); +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *facts); +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_namespace_test_capture_fail_after(int successful_allocations); +void cbm_py_namespace_test_copy_fail_after(int successful_allocations); +#endif + // Growable array push functions (arena-allocated, no individual free needed). void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def); void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call); diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 7d416923c1..71ec069c08 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -4,8 +4,10 @@ #include "lang_specs.h" // CBMLangSpec, CBMEmbeddedLangSpec, cbm_lang_spec, cbm_ts_language #include "tree_sitter/api.h" // TSNode, ts_node_* #include "foundation/constants.h" +#include "foundation/compat.h" /* CBM_TLS */ #include "extract_node_stack.h" #include // uint32_t +#include #include #include @@ -338,6 +340,490 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* Ordered Python namespace facts are a separate content-only pass. The older + * import records keep their existing graph/heuristic meaning. */ +typedef struct { + CBMExtractCtx *ctx; + CBMPyNamespaceFacts facts; + CBMPyNamespaceFailure failure; +} PyNamespaceBuilder; + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_capture_left = -1; +void cbm_py_namespace_test_capture_fail_after(int successful_allocations) { + py_namespace_capture_left = successful_allocations; +} +#endif + +static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_capture_left == 0) { + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return NULL; + } + if (py_namespace_capture_left > 0) + py_namespace_capture_left--; +#endif + void *p = cbm_arena_alloc(b->ctx->arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + +static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { + if (len == SIZE_MAX) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return NULL; + } + char *copy = py_ns_alloc(b, len + 1); + if (!copy) + return NULL; + memcpy(copy, text, len); + copy[len] = '\0'; + return copy; +} + +static bool py_ns_slice(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (ts_node_is_null(node)) + return false; + uint32_t start = ts_node_start_byte(node), end = ts_node_end_byte(node); + if (end < start || end > (uint32_t)b->ctx->source_len) + return false; + *text = b->ctx->source + start; + *len = (size_t)(end - start); + return memchr(*text, '\0', *len) == NULL; +} + +static bool py_ns_identifier_bytes(const char *s, size_t n, bool dotted) { + bool initial = true; + if (!n) + return false; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (dotted && c == '.' && !initial) { + initial = true; + continue; + } + if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (!initial && c >= '0' && c <= '9')) { + initial = false; + continue; + } + return false; + } + return !initial; +} + +static const char *py_ns_name(PyNamespaceBuilder *b, TSNode node, bool dotted) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, node, &text, &len) || !py_ns_identifier_bytes(text, len, dotted)) + return NULL; + return py_ns_copy(b, text, len); +} + +static bool py_ns_push(PyNamespaceBuilder *b, CBMPyNamespaceFact fact) { + if (b->failure) + return false; + if (b->facts.count == b->facts.cap) { + if (b->facts.cap > INT_MAX / 2) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + int cap = b->facts.cap ? b->facts.cap * 2 : 8; + if ((size_t)cap > SIZE_MAX / sizeof(*b->facts.items)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + CBMPyNamespaceFact *items = py_ns_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (b->facts.count) + memcpy(items, b->facts.items, (size_t)b->facts.count * sizeof(*items)); + b->facts.items = items; + b->facts.cap = cap; + } + b->facts.items[b->facts.count++] = fact; + return true; +} + +static bool py_ns_unknown(PyNamespaceBuilder *b, CBMPyNamespaceUnknownReason reason) { + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_UNKNOWN, .reason = reason}); +} + +static bool py_ns_plain_string(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (strcmp(ts_node_type(node), "string") != 0 || !py_ns_slice(b, node, text, len) || *len < 2) + return false; + char quote = (*text)[0]; + if ((quote != '\'' && quote != '"') || (*text)[*len - 1] != quote) + return false; + for (size_t i = 1; i + 1 < *len; i++) { + if ((*text)[i] == '\\' || (*text)[i] == quote || (*text)[i] == '\n' || (*text)[i] == '\r') + return false; + } + (*text)++; + *len -= 2; + return true; +} + +/* Bounded syntactic proof only. Unsupported or deep expressions become an + * unknown-effect barrier; no evaluator or source execution is involved. */ +static bool py_ns_safe_expr(PyNamespaceBuilder *b, TSNode node, int depth) { + if (ts_node_is_null(node)) + return true; + if (depth >= 64) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "identifier") == 0 || strcmp(kind, "integer") == 0 || + strcmp(kind, "float") == 0 || strcmp(kind, "true") == 0 || strcmp(kind, "false") == 0 || + strcmp(kind, "none") == 0 || strcmp(kind, "comment") == 0) + return true; + if (strcmp(kind, "string") == 0) { + const char *text = NULL; + size_t len = 0; + return py_ns_plain_string(b, node, &text, &len); + } + bool container = + strcmp(kind, "tuple") == 0 || strcmp(kind, "list") == 0 || + strcmp(kind, "parenthesized_expression") == 0 || strcmp(kind, "parameters") == 0 || + strcmp(kind, "default_parameter") == 0 || strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0 || strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0 || strcmp(kind, "type") == 0; + if (!container) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && !py_ns_safe_expr(b, child, depth + 1)) { + safe = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_signature_safe(PyNamespaceBuilder *b, TSNode node) { + return py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("parameters")), 0) && + py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("return_type")), 0); +} + +static bool py_ns_class_safe(PyNamespaceBuilder *b, TSNode node) { + if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("superclasses")))) + return false; + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); + if (ts_node_is_null(body)) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child)) + continue; + const char *kind = ts_node_type(child); + if (strcmp(kind, "pass_statement") == 0 || strcmp(kind, "comment") == 0) + continue; + if (strcmp(kind, "function_definition") == 0 && py_ns_signature_safe(b, child)) + continue; + if (strcmp(kind, "expression_statement") == 0 && + ts_node_named_child_count(child) == 1) { + TSNode expr = ts_node_named_child(child, 0); + if (strcmp(ts_node_type(expr), "string") == 0 && py_ns_safe_expr(b, expr, 0)) + continue; + /* Even an identifier assignment can install a descriptor whose + * __set_name__ hook executes during class creation. */ + } + safe = false; + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_import(PyNamespaceBuilder *b, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + const char *module_name = NULL; + uint32_t level = 0; + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, module, &text, &len)) + return false; + while ((size_t)level < len && text[level] == '.') + level++; + if (level == len ? level == 0 : !py_ns_identifier_bytes(text + level, len - level, true)) + return false; + module_name = py_ns_copy(b, text + level, len - level); + if (!module_name) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + int emitted = 0; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || (from && child.id == module.id) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + CBMPyNamespaceFact f = {.kind = from ? CBM_PY_NS_IMPORT_NAME : CBM_PY_NS_IMPORT_MODULE, + .module_name = module_name, + .relative_level = level}; + if (from && strcmp(kind, "wildcard_import") == 0) { + f.kind = CBM_PY_NS_IMPORT_STAR; + } else { + bool alias = strcmp(kind, "aliased_import") == 0; + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + if (!alias && strcmp(kind, "dotted_name") != 0 && strcmp(kind, "identifier") != 0) { + ok = false; + break; + } + const char *imported = py_ns_name(b, name, !from); + if (!imported) { + ok = false; + break; + } + if (from) + f.member_name = imported; + else + f.module_name = imported; + if (alias) { + f.local_name = + py_ns_name(b, ts_node_child_by_field_name(child, TS_FIELD("alias")), false); + } else if (from) { + f.local_name = imported; + } else { + size_t n = strcspn(imported, "."); + f.local_name = py_ns_copy(b, imported, n); + f.flags = CBM_PY_NS_IMPORT_BINDS_ROOT; + } + if (!f.local_name) { + ok = false; + break; + } + } + if (!py_ns_push(b, f)) { + ok = false; + break; + } + emitted++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok && emitted > 0; +} + +static bool py_ns_all_literal(PyNamespaceBuilder *b, TSNode node, CBMPyNamespaceFact *fact) { + if (ts_node_is_null(node)) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "list") != 0 && strcmp(kind, "tuple") != 0) + return false; + uint32_t count = ts_node_named_child_count(node); + if (count > INT_MAX || (size_t)count > SIZE_MAX / sizeof(*fact->names)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + const char **names = count ? py_ns_alloc(b, (size_t)count * sizeof(*names)) : NULL; + if (count && !names) + return false; + int used = 0; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *text = NULL; + size_t len = 0; + if (!py_ns_plain_string(b, child, &text, &len) || + !(names[used] = py_ns_copy(b, text, len))) { + ok = false; + break; + } + used++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + if (!ok) + return false; + fact->sequence_kind = + strcmp(kind, "list") == 0 ? CBM_PY_NS_SEQUENCE_LIST : CBM_PY_NS_SEQUENCE_TUPLE; + fact->names = used ? names : NULL; + fact->name_count = used; + return true; +} + +static bool py_ns_assignment(PyNamespaceBuilder *b, TSNode node) { + TSNode left = ts_node_child_by_field_name(node, TS_FIELD("left")); + TSNode right = ts_node_child_by_field_name(node, TS_FIELD("right")); + TSNode annotation = ts_node_child_by_field_name(node, TS_FIELD("type")); + const char *name = py_ns_name(b, left, false); + if (!name) + return false; + if (!py_ns_safe_expr(b, right, 0) || !py_ns_safe_expr(b, annotation, 0)) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + if (ts_node_is_null(right)) + return true; /* annotation alone does not bind the target */ + bool augmented = strcmp(ts_node_type(node), "augmented_assignment") == 0; + if (strcmp(name, "__all__") != 0) { + if (augmented) + return py_ns_unknown(b, CBM_PY_NS_REASON_UNSUPPORTED); + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = CBM_PY_NS_REASON_VALUE}); + } + CBMPyNamespaceFact fact = {.kind = augmented ? CBM_PY_NS_ALL_APPEND : CBM_PY_NS_ALL_SET}; + bool plus = true; + if (augmented) { + const char *op = NULL; + size_t len = 0; + plus = py_ns_slice(b, ts_node_child_by_field_name(node, TS_FIELD("operator")), &op, &len) && + len == 2 && memcmp(op, "+=", 2) == 0; + } + if (plus && py_ns_all_literal(b, right, &fact)) + return py_ns_push(b, fact); + return py_ns_push( + b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_UNKNOWN, .reason = CBM_PY_NS_REASON_VALUE}); +} + +static bool py_ns_delete(PyNamespaceBuilder *b, TSNode node, int depth) { + if (depth > 2) + return false; + if (strcmp(ts_node_type(node), "identifier") == 0) { + const char *name = py_ns_name(b, node, false); + if (!name) + return false; + return py_ns_push(b, + strcmp(name, "__all__") == 0 + ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_DELETE} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_DELETE, .local_name = name}); + } + const char *kind = ts_node_type(node); + if (strcmp(kind, "delete_statement") != 0 && strcmp(kind, "expression_list") != 0) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && strcmp(ts_node_type(child), "comment") != 0 && + !py_ns_delete(b, child, depth + 1)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { + const char *kind = ts_node_type(node); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) + return true; + if (strcmp(kind, "import_statement") == 0) + return py_ns_import(b, node, false); + if (strcmp(kind, "import_from_statement") == 0) + return py_ns_import(b, node, true); + bool decorated = strcmp(kind, "decorated_definition") == 0; + TSNode def = decorated ? ts_node_child_by_field_name(node, TS_FIELD("definition")) : node; + if (!ts_node_is_null(def)) { + const char *dk = ts_node_type(def); + bool function = strcmp(dk, "function_definition") == 0; + bool class_def = strcmp(dk, "class_definition") == 0; + if (function || class_def) { + const char *name = + py_ns_name(b, ts_node_child_by_field_name(def, TS_FIELD("name")), false); + if (!name) + return false; + bool proven = !decorated && (function || py_ns_class_safe(b, def)); + if (!proven || (function && !py_ns_signature_safe(b, def))) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + return py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated + ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + } + } + if (strcmp(kind, "delete_statement") == 0) + return py_ns_delete(b, node, 0); + if (strcmp(kind, "expression_statement") == 0 && ts_node_named_child_count(node) == 1) { + TSNode expr = ts_node_named_child(node, 0); + const char *ek = ts_node_type(expr); + if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) + return py_ns_assignment(b, expr); + return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + } + if (strcmp(kind, "if_statement") == 0 || strcmp(kind, "try_statement") == 0 || + strcmp(kind, "for_statement") == 0 || strcmp(kind, "while_statement") == 0 || + strcmp(kind, "with_statement") == 0 || strcmp(kind, "match_statement") == 0) + return py_ns_unknown(b, CBM_PY_NS_REASON_COMPOUND); + return false; +} + +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result) + return CBM_PY_NS_INCOMPLETE; + PyNamespaceBuilder b = {.ctx = ctx, + .facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_COMPLETE}}; + if (ctx->language != CBM_LANG_PYTHON) { + b.facts.status = CBM_PY_NS_NOT_APPLICABLE; + } else if (!ctx->arena || !ctx->source || ctx->source_len < 0) { + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else if (ts_node_is_null(ctx->root) || ts_node_has_error(ctx->root)) { + py_ns_unknown(&b, CBM_PY_NS_REASON_PARSE); + } else { + TSTreeCursor cursor = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(node)) + continue; + int start = b.facts.count; + if (!py_ns_statement(&b, node) && !b.failure) { + b.facts.count = start; + py_ns_unknown(&b, CBM_PY_NS_REASON_UNSUPPORTED); + } + if (b.failure) + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + } + if (!b.failure && !cbm_py_namespace_facts_valid(&b.facts)) + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + if (b.failure) { + b.facts = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = b.failure}; + } + ctx->result->py_namespace = b.facts; + return b.facts.status; +} + // --- ES module imports (JS/TS/TSX) --- // import X from "Y"; import {A, B} from "Y"; import * as X from "Y" // const X = require("Y") diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 82f661d053..9bb92fa72f 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -299,6 +299,54 @@ static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { } } +/* Facts have counted lists, not sentinel arrays. Bail before touching a failed + * array copy so no string rewrite can land in the original result's storage. */ +static bool cr_py_size(cr_ctx_t *c, size_t size) { + if (c->failed) + return false; + if (c->phase == CR_MEASURE && + (size > SIZE_MAX - CR_ALIGN || c->bytes > SIZE_MAX - cr_aligned(size))) { + c->failed = true; + return false; + } + return true; +} + +static bool cr_py_string(cr_ctx_t *c, const char **field) { + if (!*field) + return true; + if (c->phase == CR_COUNT && c->refs == SIZE_MAX) { + c->failed = true; + return false; + } + if (c->phase == CR_MEASURE && !cr_py_size(c, strlen(*field) + 1)) + return false; + cr_str(c, field); + return !c->failed; +} + +static void cr_walk_py_namespace(cr_ctx_t *c, CBMPyNamespaceFacts *facts) { + if (!cr_py_size(c, (size_t)facts->count * sizeof(*facts->items))) + return; + cr_array(c, (void **)&facts->items, facts->count, sizeof(*facts->items)); + if (c->failed) + return; + for (int i = 0; i < facts->count; i++) { + CBMPyNamespaceFact *f = &facts->items[i]; + if (!cr_py_string(c, &f->local_name) || !cr_py_string(c, &f->module_name) || + !cr_py_string(c, &f->member_name) || + !cr_py_size(c, (size_t)f->name_count * sizeof(*f->names))) + return; + cr_array(c, (void **)&f->names, f->name_count, sizeof(*f->names)); + if (c->failed) + return; + for (int j = 0; j < f->name_count; j++) { + if (!cr_py_string(c, &f->names[j])) + return; + } + } +} + static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition)); for (int i = 0; i < r->defs.count && r->defs.items; i++) { @@ -392,6 +440,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->error_msg); cr_str(c, &r->error_ranges); cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); + cr_walk_py_namespace(c, &r->py_namespace); } void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) { @@ -474,7 +523,8 @@ static void cr_scratch_put(cr_ctx_t *c) { } void cbm_result_compact(CBMFileResult *result) { - if (!result || result->arena.nblocks == 0) { + if (!result || result->arena.nblocks == 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return; } cr_ctx_t c; @@ -486,14 +536,23 @@ void cbm_result_compact(CBMFileResult *result) { c.phase = CR_COUNT; cr_walk(&c, &tmp); + /* Bound intern-table/replay sizing before the existing power-of-two step. */ + if (c.failed || c.refs > (SIZE_MAX / (sizeof(cr_slot_t) * 8) - CR_MIN_TABLE)) + return; c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE); + if (c.cap > UINT32_MAX) /* replay entries store slot indexes in uint32_t */ + return; if (!cr_scratch_get(&c)) { return; } c.phase = CR_MEASURE; cr_walk(&c, &tmp); + if (c.failed) { + cr_scratch_put(&c); + return; + } for (size_t i = 0; i < c.cap; i++) { c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */ } @@ -518,6 +577,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.defs.cap = tmp.defs.count; tmp.calls.cap = tmp.calls.count; tmp.imports.cap = tmp.imports.count; + tmp.py_namespace.cap = tmp.py_namespace.count; tmp.usages.cap = tmp.usages.count; tmp.throws.cap = tmp.throws.count; tmp.rw.cap = tmp.rw.count; diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c index 33a732451d..503e1fb2a2 100644 --- a/internal/cbm/result_spill.c +++ b/internal/cbm/result_spill.c @@ -258,7 +258,8 @@ bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFile * result that owns sub-results (embedded languages) has more arenas than * that and stays in memory. A retained parse tree is a re-parse cache, * not data: it is dropped with the in-memory result below. */ - if (result->arena.nblocks != 1 || result->owned_result_count != 0) { + if (result->arena.nblocks != 1 || result->owned_result_count != 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return false; } /* The disk ceiling. Refusing a park leaves the result in memory, which is @@ -356,6 +357,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) { } cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len, r->arena.blocks[0]); + if (!cbm_py_namespace_facts_valid(&r->py_namespace)) { + cbm_free_result(r); + return NULL; + } atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed); return r; } diff --git a/internal/cbm/result_spill.h b/internal/cbm/result_spill.h index 88cfd37205..8777a5bca0 100644 --- a/internal/cbm/result_spill.h +++ b/internal/cbm/result_spill.h @@ -43,7 +43,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot); /* Read only the parked header of slot `slot` into *out: every count in it * is valid, every pointer meaningless. False when the slot is empty or the - * read fails. Lets a consumer skip the load when there is nothing to read. */ + * read fails. Lets a consumer skip the load when there is nothing to read. + * Python namespace version/language/status/count are scalar evidence only: + * zero facts with NOT_CAPTURED/INCOMPLETE is not a known-empty namespace, and + * event pointers require load. Failed peek/load provides no namespace evidence. */ bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out); /* Read only the parked header of slot `slot`: how many defs and impl diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index 4c3cde3363..5bb52a55ef 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -19,6 +19,7 @@ #include "pipeline/lsp_surface.h" #include "pipeline/pipeline_internal.h" +#include #include #include #include @@ -29,7 +30,7 @@ #include "pipeline/worker_pool.h" #include "yyjson/yyjson.h" -enum { SURFACE_CODEC_VERSION = 1 }; +enum { SURFACE_CODEC_VERSION = 2 }; /* Labels the incremental name registry serves that pxc_map_label does NOT * carry into the CBMLSPDef set. Their (name, qn, label) triple must still @@ -71,10 +72,288 @@ static void add_str_array_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, cons yyjson_mut_obj_add_val(doc, obj, key, arr); } +/* The namespace payload is content-only. Its ordered records never depend on + * the registry used to build ordinary LSP definitions. */ +static bool surface_py_add_nullable(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, + const char *value) { + return value ? yyjson_mut_obj_add_str(doc, obj, key, value) + : yyjson_mut_obj_add_null(doc, obj, key); +} + +static bool surface_add_python_namespace(yyjson_mut_doc *doc, yyjson_mut_val *root, + const CBMFileResult *result, CBMLanguage language, + const char *rel_path) { + if (language != CBM_LANG_PYTHON) { + return yyjson_mut_obj_add_null(doc, root, "py"); + } + if (!result || !result->module_qn || !result->module_qn[0] || !rel_path || !rel_path[0] || + result->py_namespace.status != CBM_PY_NS_COMPLETE || + result->py_namespace.language != CBM_LANG_PYTHON || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { + return false; + } + const CBMPyNamespaceFacts *facts = &result->py_namespace; + yyjson_mut_val *py = yyjson_mut_obj(doc); + yyjson_mut_val *events = yyjson_mut_arr(doc); + if (!py || !events || !yyjson_mut_obj_add_uint(doc, py, "v", facts->version) || + !yyjson_mut_obj_add_int(doc, py, "lang", (int)facts->language) || + !yyjson_mut_obj_add_str(doc, py, "module", result->module_qn) || + !yyjson_mut_obj_add_str(doc, py, "path", rel_path) || + !yyjson_mut_obj_add_int(doc, py, "status", (int)facts->status) || + !yyjson_mut_obj_add_int(doc, py, "failure", (int)facts->failure)) { + return false; + } + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + yyjson_mut_val *event = yyjson_mut_obj(doc); + yyjson_mut_val *names = yyjson_mut_arr(doc); + if (!event || !names || !yyjson_mut_obj_add_int(doc, event, "k", (int)f->kind) || + !yyjson_mut_obj_add_int(doc, event, "d", (int)f->def_kind) || + !yyjson_mut_obj_add_int(doc, event, "s", (int)f->sequence_kind) || + !yyjson_mut_obj_add_int(doc, event, "r", (int)f->reason) || + !yyjson_mut_obj_add_uint(doc, event, "f", f->flags) || + !yyjson_mut_obj_add_uint(doc, event, "l", f->relative_level) || + !surface_py_add_nullable(doc, event, "n", f->local_name) || + !surface_py_add_nullable(doc, event, "m", f->module_name) || + !surface_py_add_nullable(doc, event, "i", f->member_name)) { + return false; + } + for (int j = 0; j < f->name_count; j++) { + if (!yyjson_mut_arr_add_str(doc, names, f->names[j])) { + return false; + } + } + if (!yyjson_mut_obj_add_val(doc, event, "a", names) || + !yyjson_mut_arr_add_val(events, event)) { + return false; + } + } + return yyjson_mut_obj_add_val(doc, py, "events", events) && + yyjson_mut_obj_add_val(doc, root, "py", py); +} + +/* Reject duplicate fields rather than letting object lookup choose one. */ +static yyjson_val *surface_unique_field(yyjson_val *obj, const char *name) { + if (!yyjson_is_obj(obj)) { + return NULL; + } + yyjson_val *found = NULL; + yyjson_obj_iter it = yyjson_obj_iter_with(obj); + yyjson_val *key; + while ((key = yyjson_obj_iter_next(&it))) { + const char *text = yyjson_get_str(key); + if (text && yyjson_get_len(key) == strlen(name) && strcmp(text, name) == 0) { + if (found) { + return NULL; + } + found = yyjson_obj_iter_get_val(key); + } + } + return found; +} + +static bool surface_py_uint(yyjson_val *value, uint32_t maximum, uint32_t *out) { + if (!yyjson_is_int(value)) { + return false; + } + int64_t n = yyjson_get_sint(value); + if (n < 0 || (uint64_t)n > maximum) { + return false; + } + *out = (uint32_t)n; + return true; +} + +static bool surface_py_string(yyjson_val *value, bool nullable, const char **out) { + *out = NULL; + if (!value) { + return false; + } + if (yyjson_is_null(value)) { + return nullable; + } + const char *text = yyjson_get_str(value); + if (!text || strlen(text) != yyjson_get_len(value)) { + return false; + } + *out = text; + return true; +} + +/* Arrays belong to scratch; strings borrow the immutable JSON document. + * The caller keeps both alive until validation/copy/comparison is finished. */ +static int surface_python_namespace_parse(CBMArena *scratch, yyjson_val *root, + cbm_lsp_python_namespace_t *out) { + memset(out, 0, sizeof(*out)); + uint32_t version = 0, file_language = 0; + yyjson_val *py = surface_unique_field(root, "py"); + if (!surface_py_uint(surface_unique_field(root, "v"), UINT32_MAX, &version) || + version != SURFACE_CODEC_VERSION || !py || + !surface_py_uint(surface_unique_field(root, "lang"), CBM_LANG_COUNT - 1, &file_language) || + !yyjson_is_arr(surface_unique_field(root, "lsp"))) { + return -1; + } + if (yyjson_is_null(py)) { + return file_language == CBM_LANG_PYTHON ? -1 : 0; + } + if (file_language != CBM_LANG_PYTHON || !yyjson_is_obj(py) || yyjson_obj_size(py) != 7) { + return -1; + } + CBMPyNamespaceFacts *facts = &out->facts; + uint32_t language, status, failure; + if (!surface_py_uint(surface_unique_field(py, "v"), UINT32_MAX, &facts->version) || + !surface_py_uint(surface_unique_field(py, "lang"), INT_MAX, &language) || + !surface_py_uint(surface_unique_field(py, "status"), INT_MAX, &status) || + !surface_py_uint(surface_unique_field(py, "failure"), INT_MAX, &failure) || + !surface_py_string(surface_unique_field(py, "module"), false, &out->module_qn) || + !surface_py_string(surface_unique_field(py, "path"), false, &out->rel_path) || + !out->module_qn[0] || !out->rel_path[0] || language != CBM_LANG_PYTHON || + status != CBM_PY_NS_COMPLETE || failure != CBM_PY_NS_FAILURE_NONE) { + return -1; + } + facts->language = (CBMLanguage)language; + facts->status = (CBMPyNamespaceStatus)status; + facts->failure = (CBMPyNamespaceFailure)failure; + yyjson_val *events = surface_unique_field(py, "events"); + if (!yyjson_is_arr(events)) { + return -1; + } + size_t count = yyjson_arr_size(events); + if (count > INT_MAX || count > SIZE_MAX / sizeof(*facts->items)) { + return -1; + } + facts->count = (int)count; + facts->cap = (int)count; + if (count) { + facts->items = cbm_arena_alloc(scratch, count * sizeof(*facts->items)); + if (!facts->items) { + return -1; + } + memset(facts->items, 0, count * sizeof(*facts->items)); + } + yyjson_arr_iter it = yyjson_arr_iter_with(events); + yyjson_val *event; + int i = 0; + while ((event = yyjson_arr_iter_next(&it))) { + if (!yyjson_is_obj(event) || yyjson_obj_size(event) != 10) { + return -1; + } + CBMPyNamespaceFact *f = &facts->items[i++]; + uint32_t kind, def_kind, sequence_kind, reason; + if (!surface_py_uint(surface_unique_field(event, "k"), INT_MAX, &kind) || + !surface_py_uint(surface_unique_field(event, "d"), INT_MAX, &def_kind) || + !surface_py_uint(surface_unique_field(event, "s"), INT_MAX, &sequence_kind) || + !surface_py_uint(surface_unique_field(event, "r"), INT_MAX, &reason) || + !surface_py_uint(surface_unique_field(event, "f"), UINT32_MAX, &f->flags) || + !surface_py_uint(surface_unique_field(event, "l"), UINT32_MAX, &f->relative_level) || + !surface_py_string(surface_unique_field(event, "n"), true, &f->local_name) || + !surface_py_string(surface_unique_field(event, "m"), true, &f->module_name) || + !surface_py_string(surface_unique_field(event, "i"), true, &f->member_name)) { + return -1; + } + f->kind = (CBMPyNamespaceFactKind)kind; + f->def_kind = (CBMPyNamespaceDefKind)def_kind; + f->sequence_kind = (CBMPyNamespaceSequenceKind)sequence_kind; + f->reason = (CBMPyNamespaceUnknownReason)reason; + yyjson_val *names = surface_unique_field(event, "a"); + if (!yyjson_is_arr(names)) { + return -1; + } + size_t n = yyjson_arr_size(names); + if (n > INT_MAX || n > SIZE_MAX / sizeof(*f->names)) { + return -1; + } + f->name_count = (int)n; + if (n) { + f->names = cbm_arena_alloc(scratch, n * sizeof(*f->names)); + if (!f->names) { + return -1; + } + } + yyjson_arr_iter ni = yyjson_arr_iter_with(names); + yyjson_val *name; + int j = 0; + while ((name = yyjson_arr_iter_next(&ni))) { + if (!surface_py_string(name, false, &f->names[j++])) { + return -1; + } + } + } + return cbm_py_namespace_facts_valid(facts) ? 1 : -1; +} + +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out) { + if (!out) { + return -1; + } + memset(out, 0, sizeof(*out)); + if (!arena || !json) { + return -1; + } + yyjson_doc *doc = yyjson_read(json, strlen(json), 0); + if (!doc) { + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t parsed = {0}, result = {0}; + int status = surface_python_namespace_parse(&scratch, yyjson_doc_get_root(doc), &parsed); + if (status == 1) { + if (!cbm_py_namespace_facts_copy(arena, &parsed.facts, &result.facts)) { + status = -1; + } else { + result.module_qn = cbm_arena_strdup(arena, parsed.module_qn); + result.rel_path = cbm_arena_strdup(arena, parsed.rel_path); + if (!result.module_qn || !result.rel_path) { + status = -1; + } + } + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(doc); + if (status == 1) { + *out = result; + } + return status; +} + +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json) { + if (!old_json) { + return -1; + } + yyjson_doc *old_doc = yyjson_read(old_json, strlen(old_json), 0); + yyjson_doc *new_doc = new_json ? yyjson_read(new_json, strlen(new_json), 0) : NULL; + if (!old_doc || (new_json && !new_doc)) { + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t old_facts, new_facts; + int old_state = + surface_python_namespace_parse(&scratch, yyjson_doc_get_root(old_doc), &old_facts); + int new_state = + new_doc ? surface_python_namespace_parse(&scratch, yyjson_doc_get_root(new_doc), &new_facts) + : 0; + int result = -1; + if (old_state >= 0 && new_state >= 0) { + result = (old_state == 0 && new_state == 0) ? 0 + : (old_state != new_state || !new_json || strcmp(old_json, new_json) != 0) ? 1 + : 0; + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return result; +} + /* Serialize one file's surface: its slice of all_defs plus the registry-only * symbols from its raw extraction defs. Returns a malloc'd JSON string and its * length. */ -static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *defs, int def_count, +static char *surface_file_to_json(const CBMFileResult *result, CBMLanguage language, + const char *rel_path, const CBMLSPDef *defs, int def_count, size_t *out_len) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); if (!doc) { @@ -82,7 +361,12 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * } yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION); + if ((unsigned)language >= CBM_LANG_COUNT || + !yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION) || + !yyjson_mut_obj_add_int(doc, root, "lang", (int)language)) { + yyjson_mut_doc_free(doc); + return NULL; + } yyjson_mut_val *lsp = yyjson_mut_arr(doc); for (int i = 0; i < def_count; i++) { @@ -150,6 +434,10 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * yyjson_mut_obj_add_val(doc, root, "http", http); } + if (!surface_add_python_namespace(doc, root, result, language, rel_path)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *json = yyjson_mut_write(doc, 0, out_len); yyjson_mut_doc_free(doc); return json; @@ -186,8 +474,9 @@ static void surface_row_one(int i, void *arg) { int start = job->def_starts ? job->def_starts[i] : 0; int end = job->def_starts ? job->def_starts[i + 1] : 0; size_t json_len = 0; - char *json = surface_file_to_json(fr, job->all_defs ? job->all_defs + start : NULL, end - start, - &json_len); + char *json = + surface_file_to_json(fr, job->files[i].language, job->files[i].rel_path, + job->all_defs ? job->all_defs + start : NULL, end - start, &json_len); cbm_pipeline_result_release(fr, loaded); if (!json) { atomic_store_explicit(&job->failed, true, memory_order_relaxed); @@ -302,9 +591,14 @@ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLS return -1; } yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *ver = root ? yyjson_obj_get(root, "v") : NULL; - yyjson_val *lsp = root ? yyjson_obj_get(root, "lsp") : NULL; - if (!ver || yyjson_get_int(ver) != SURFACE_CODEC_VERSION || !lsp || !yyjson_is_arr(lsp)) { + CBMArena namespace_scratch; + cbm_arena_init(&namespace_scratch); + cbm_lsp_python_namespace_t namespace_payload = {0}; + int namespace_status = + surface_python_namespace_parse(&namespace_scratch, root, &namespace_payload); + cbm_arena_destroy(&namespace_scratch); + yyjson_val *lsp = surface_unique_field(root, "lsp"); + if (namespace_status < 0) { yyjson_doc_free(doc); return -1; } diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h index de3390d85e..dc198bb654 100644 --- a/src/pipeline/lsp_surface.h +++ b/src/pipeline/lsp_surface.h @@ -10,8 +10,10 @@ * unchanged — a body edit — and skip recomputing its dependents, and * (b) rehydrate the cross registries for files it does not re-parse. * - * The JSON is a versioned object {"v":1,"lsp":[...],"reg":[...]}. Field - * order and array order are fixed by the writer, so byte equality of the + * The JSON is a versioned object {"v":2,"lang":N,"lsp":[...],"reg":[...],"py":...}. + * Python rows carry content-only namespace facts, including ordered binding + * events and unknown effects; other languages carry an explicit null. + * Field order and array order are fixed by the writer, so byte equality of the * serialization is surface equality; the sha256 of these bytes is the * stored surface_sha. Rows written by a different codec version fail the * "v" check on load and route the run to a full rebuild. @@ -28,7 +30,8 @@ * all_defs[def_starts[i] .. def_starts[i+1]) are file i's defs. Rows carry * heap strings; release with cbm_store_free_lsp_surfaces. Files with an * empty surface still get a row (empty arrays hash too — "no defs" must be - * distinguishable from "no data"). Returns 0, or -1 on allocation failure. */ + * distinguishable from "no data"). Returns 0, or -1 on allocation failure + * or invalid/incomplete Python namespace metadata. */ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project, CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const CBMLSPDef *all_defs, const int *def_starts, @@ -40,4 +43,29 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec * different codec version — callers route that to a full rebuild. */ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs); +/* A restored Python namespace retains its raw module identity and source path. + * Facts and origin strings live in the caller arena; no JSON storage is borrowed. */ +typedef struct { + const char *module_qn; + const char *rel_path; + CBMPyNamespaceFacts facts; +} cbm_lsp_python_namespace_t; + +/* Returns 1 for a complete Python payload, 0 for a valid non-Python row, or + * -1 for invalid/incomplete metadata or allocation failure. Output is empty + * unless a complete Python payload was copied successfully. */ +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out); + +/* Conservative namespace admission for incremental planning. Both rows must + * have a current, valid fact envelope before any equality shortcut is used. + * Returns -1 for invalid/incomplete metadata, 0 when both rows are non-Python + * or their complete Python surfaces are identical, and 1 for a Python surface + * change or a language transition involving Python. A NULL new_json denotes + * deletion: valid Python metadata returns 1, valid non-Python metadata 0. + * NULL old_json is invalid. The full Python surface comparison deliberately + * includes definition properties until a narrower dependency closure is proven. + */ +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json); + #endif /* CBM_PIPELINE_LSP_SURFACE_H */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658d..0e06e8032c 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1817,8 +1817,9 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } - /* Early cutoff per changed file; surface-changed files seed the - * dependent query. Deleted files always seed it. */ + /* Validate Python namespace metadata before any hash cutoff or dependent + * query. Python changes require a full rebuild; other surface changes and + * deletions seed the dependent query. */ int n_surface_changed = 0; const char **dep_targets = (const char **)calloc((size_t)(n_changed + n_deleted), sizeof(char *)); @@ -1843,6 +1844,15 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p decline = "missing_surface_row"; goto done; } + int python_change = + cbm_lsp_surface_python_change(stored_row->defs_json, fresh_row->defs_json); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } if (strcmp(stored_row->surface_sha, fresh_row->surface_sha) == 0) { continue; /* body edit: the file re-resolves, nobody else does */ } @@ -1858,6 +1868,17 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p dep_targets[dep_target_count++] = changed_paths[i]; } for (int i = 0; i < n_deleted; i++) { + const cbm_lsp_surface_row_t *stored_row = + cbm_ht_get(rows_by_path, changed_paths[n_changed + i]); + int python_change = + cbm_lsp_surface_python_change(stored_row ? stored_row->defs_json : NULL, NULL); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } dep_targets[dep_target_count++] = changed_paths[n_changed + i]; } cbm_ht_free(rows_by_path); diff --git a/src/store/store.h b/src/store/store.h index 06ea1fbb7e..12e4e6483e 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -23,7 +23,7 @@ typedef struct cbm_store cbm_store_t; #define CBM_STORE_OK 0 #define CBM_STORE_ERR (-1) #define CBM_STORE_NOT_FOUND (-2) -#define CBM_INDEX_FORMAT_VERSION 1 +#define CBM_INDEX_FORMAT_VERSION 2 #define CBM_STORE_CANCELLED (-3) #define CBM_STORE_SCAN_LIMIT (-4) #define CBM_STORE_CALLBACK_ERR (-5) @@ -112,10 +112,11 @@ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *ou * stores that have no File nodes. Caller frees each out[i] and out itself. */ int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count); -/* Persisted index-format identity. Bump when a change alters the QN scheme - * or node identity of an already-written graph, so an old DB is routed - * through the full-reindex path instead of producing a mixed graph. - * 1 = File QNs keep the file extension (#769). */ +/* Persisted index-format identity. Bump when a change alters the QN scheme, + * node identity, or required persisted semantic metadata, so an old DB is + * routed through the full-reindex path before incremental/no-op admission. + * 1 = File QNs keep the file extension (#769). + * 2 = Python namespace facts use the versioned LSP surface contract. */ int cbm_store_get_format_version(cbm_store_t *s, int *out); int cbm_store_set_format_version(cbm_store_t *s, int version); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index cf35289210..efbeac315b 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8964,6 +8964,485 @@ TEST(extract_enclosing_sweep_matches_scan_issue1527) { PASS(); } +/* Python namespace facts: provenance is ordered, content-only and independently + * owned. These tests do not enable any reexport resolution. */ +static bool py_facts_equal(const CBMPyNamespaceFacts *a, const CBMPyNamespaceFacts *b) { + if (!cbm_py_namespace_facts_valid(a) || !cbm_py_namespace_facts_valid(b) || + a->version != b->version || a->language != b->language || a->status != b->status || + a->failure != b->failure || a->count != b->count) + return false; + for (int i = 0; i < a->count; i++) { + const CBMPyNamespaceFact *x = &a->items[i], *y = &b->items[i]; + if (x->kind != y->kind || x->def_kind != y->def_kind || + x->sequence_kind != y->sequence_kind || x->reason != y->reason || + x->flags != y->flags || x->relative_level != y->relative_level || + x->name_count != y->name_count || !cmp_str_eq(x->local_name, y->local_name) || + !cmp_str_eq(x->module_name, y->module_name) || + !cmp_str_eq(x->member_name, y->member_name)) + return false; + for (int j = 0; j < x->name_count; j++) { + if (!cmp_str_eq(x->names[j], y->names[j])) + return false; + } + } + return true; +} + +static bool py_facts_owned_range(const CBMArena *arena, const void *ptr, size_t bytes) { + if (!ptr) + return bytes == 0; + uintptr_t address = (uintptr_t)ptr; + for (int i = 0; i < arena->nblocks; i++) { + uintptr_t start = (uintptr_t)arena->blocks[i]; + size_t size = arena->block_sizes[i]; + if (address >= start && bytes <= size && address - start <= size - bytes) + return true; + } + return false; +} + +static bool py_facts_owned_string(const CBMArena *arena, const char *str) { + return !str || (py_facts_owned_range(arena, str, 1) && + py_facts_owned_range(arena, str, strlen(str) + 1)); +} + +static bool py_facts_owned(const CBMArena *arena, const CBMPyNamespaceFacts *facts) { + if (!py_facts_owned_range(arena, facts->items, (size_t)facts->count * sizeof(*facts->items))) + return false; + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + if (!py_facts_owned_string(arena, f->local_name) || + !py_facts_owned_string(arena, f->module_name) || + !py_facts_owned_string(arena, f->member_name) || + !py_facts_owned_range(arena, f->names, (size_t)f->name_count * sizeof(*f->names))) + return false; + for (int j = 0; j < f->name_count; j++) { + if (!py_facts_owned_string(arena, f->names[j])) + return false; + } + } + return true; +} + +static const char PY_FACTS_ORDERED[] = "def item():\n return 1\n" + "from first import item\n" + "from second import item as item\n" + "item = unknown\n" + "del item\n" + "import pkg.mod\n" + "import pkg.mod as alias\n" + "from . import child as local\n" + "from ..outer import member\n" + "from one.shared import *\n" + "from two.shared import *\n" + "class Local:\n pass\n"; + +TEST(python_namespace_facts_preserve_order_and_import_provenance) { + CBMFileResult *r = extract(PY_FACTS_ORDERED, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 12 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "item") && + has_def(r, "Class", "Local") && r->module_qn && + strcmp(r->module_qn, "t.pkg.__init__") == 0; + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_OWN_DEF && f[0].def_kind == CBM_PY_NS_DEF_FUNCTION && + strcmp(f[0].local_name, "item") == 0 && f[1].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[1].module_name, "first") == 0 && strcmp(f[1].member_name, "item") == 0 && + strcmp(f[1].local_name, "item") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].module_name, "second") == 0 && strcmp(f[2].local_name, "item") == 0 && + f[3].kind == CBM_PY_NS_SHADOW && strcmp(f[3].local_name, "item") == 0 && + f[4].kind == CBM_PY_NS_DELETE && strcmp(f[4].local_name, "item") == 0 && + f[5].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[5].module_name, "pkg.mod") == 0 && + strcmp(f[5].local_name, "pkg") == 0 && f[5].flags == CBM_PY_NS_IMPORT_BINDS_ROOT && + f[6].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[6].module_name, "pkg.mod") == 0 && + strcmp(f[6].local_name, "alias") == 0 && f[6].flags == 0 && f[7].relative_level == 1 && + strcmp(f[7].module_name, "") == 0 && strcmp(f[7].member_name, "child") == 0 && + strcmp(f[7].local_name, "local") == 0 && f[8].relative_level == 2 && + strcmp(f[8].module_name, "outer") == 0 && strcmp(f[8].member_name, "member") == 0 && + f[9].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[9].module_name, "one.shared") == 0 && + f[10].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[10].module_name, "two.shared") == 0 && + f[11].kind == CBM_PY_NS_OWN_DEF && f[11].def_kind == CBM_PY_NS_DEF_CLASS && + strcmp(f[11].local_name, "Local") == 0 && py_facts_owned(&r->arena, &r->py_namespace); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { + static const char source[] = "__all__ = ['old']; __all__ = []\n" + "__all__ = dynamic\n" + "__all__ = ('new', '_private', 'a,b', '')\n" + "__all__ += ('new',)\n" + "__all__ -= []\n" + "del __all__\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *empty = extract("# no bindings\n", CBM_LANG_PYTHON, "t", "empty.py"); + bool ok = r && empty && !r->has_error && !r->parse_incomplete && !empty->has_error && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 7 && + empty->py_namespace.status == CBM_PY_NS_COMPLETE && empty->py_namespace.count == 0 && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_ALL_SET && f[0].name_count == 1 && + strcmp(f[0].names[0], "old") == 0 && f[1].kind == CBM_PY_NS_ALL_SET && + f[1].name_count == 0 && f[1].names == NULL && + f[1].sequence_kind == CBM_PY_NS_SEQUENCE_LIST && f[2].kind == CBM_PY_NS_ALL_UNKNOWN && + f[3].kind == CBM_PY_NS_ALL_SET && f[3].sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && + f[3].name_count == 4 && strcmp(f[3].names[0], "new") == 0 && + strcmp(f[3].names[1], "_private") == 0 && strcmp(f[3].names[2], "a,b") == 0 && + strcmp(f[3].names[3], "") == 0 && f[4].kind == CBM_PY_NS_ALL_APPEND && + f[4].name_count == 1 && strcmp(f[4].names[0], "new") == 0 && + f[5].kind == CBM_PY_NS_ALL_UNKNOWN && f[6].kind == CBM_PY_NS_ALL_DELETE; + } + if (r) + cbm_free_result(r); + if (empty) + cbm_free_result(empty); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unknown_effects_and_body_stability) { + static const char one[] = "from pkg import Item\ndef f():\n return 1\n"; + static const char two[] = + "from pkg import Item\ndef f():\n from elsewhere import Other\n return Other()\n"; + CBMFileResult *a = extract(one, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *b = extract(two, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = a && b && !a->has_error && !b->has_error && !a->parse_incomplete && + !b->parse_incomplete && a->py_namespace.count == 2 && + py_facts_equal(&a->py_namespace, &b->py_namespace); + if (a) + cbm_free_result(a); + if (b) + cbm_free_result(b); + static const char unknown[] = "from pkg import Item\n" + "if enabled:\n from other import Item\n" + "@deco\ndef wrapped():\n pass\n" + "class Derived(Base):\n pass\n" + "Item = factory()\n"; + a = extract(unknown, CBM_LANG_PYTHON, "t", "main.py"); + ok = a && !a->has_error && !a->parse_incomplete && a->py_namespace.count == 8 && ok; + if (a && a->py_namespace.count == 8) { + const CBMPyNamespaceFact *f = a->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_NAME && f[1].kind == CBM_PY_NS_UNKNOWN && + f[1].reason == CBM_PY_NS_REASON_COMPOUND && f[2].kind == CBM_PY_NS_UNKNOWN && + f[3].kind == CBM_PY_NS_SHADOW && f[3].reason == CBM_PY_NS_REASON_DECORATED && + strcmp(f[3].local_name, "wrapped") == 0 && f[4].kind == CBM_PY_NS_UNKNOWN && + f[5].kind == CBM_PY_NS_SHADOW && f[5].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && + f[6].kind == CBM_PY_NS_UNKNOWN && f[7].kind == CBM_PY_NS_SHADOW && ok; + } + if (a) + cbm_free_result(a); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_parse_errors_are_complete_unknown) { + const char *source = "from pkg import Before\ndef broken(:\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && r->parse_incomplete && r->py_namespace.status == CBM_PY_NS_COMPLETE && + r->py_namespace.count == 1 && r->py_namespace.items[0].kind == CBM_PY_NS_UNKNOWN && + r->py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (r) + cbm_free_result(r); + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + CBMExtractCtx ctx = {.arena = &fallback.arena, + .result = &fallback, + .source = "", + .source_len = 0, + .language = CBM_LANG_PYTHON}; + ok = cbm_extract_python_namespace_facts(&ctx) == CBM_PY_NS_COMPLETE && + fallback.py_namespace.count == 1 && + fallback.py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && ok; + cbm_arena_destroy(&fallback.arena); + r = extract("const x = 1;\n", CBM_LANG_JAVASCRIPT, "t", "__init__.js"); + ok = r && !r->has_error && r->py_namespace.status == CBM_PY_NS_NOT_APPLICABLE && + r->py_namespace.language == CBM_LANG_JAVASCRIPT && r->py_namespace.count == 0 && ok; + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_capture_and_copy_fail_atomically) { + static const char source[] = "from .pkg import Item as Local\n__all__ = ['Local', '']\n"; + bool ok = true, capture_complete = false, copy_complete = false; + int capture_failures = 0, copy_failures = 0; + for (int limit = 0; limit < 64 && !capture_complete; limit++) { + cbm_py_namespace_test_capture_fail_after(limit); + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + cbm_py_namespace_test_capture_fail_after(-1); + if (!r) { + ok = false; + break; + } + const CBMPyNamespaceFacts *f = &r->py_namespace; + capture_complete = f->status == CBM_PY_NS_COMPLETE; + if (capture_complete) { + ok = f->count == 2 && cbm_py_namespace_facts_valid(f) && ok; + } else { + capture_failures++; + ok = f->status == CBM_PY_NS_INCOMPLETE && f->failure == CBM_PY_NS_FAILURE_ALLOCATION && + !f->items && f->count == 0 && f->cap == 0 && cbm_py_namespace_facts_valid(f) && ok; + } + cbm_free_result(r); + } + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena reference_arena; + cbm_arena_init(&reference_arena); + CBMPyNamespaceFacts original = {0}; + ok = r && cbm_py_namespace_facts_copy(&reference_arena, &r->py_namespace, &original) && ok; + for (int limit = 0; r && limit < 64 && !copy_complete; limit++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + cbm_py_namespace_test_copy_fail_after(limit); + copy_complete = cbm_py_namespace_facts_copy(&arena, &r->py_namespace, ©); + cbm_py_namespace_test_copy_fail_after(-1); + if (copy_complete) { + ok = py_facts_equal(©, &r->py_namespace) && py_facts_owned(&arena, ©) && ok; + } else { + copy_failures++; + ok = copy.status == CBM_PY_NS_INCOMPLETE && + copy.failure == CBM_PY_NS_FAILURE_ALLOCATION && !copy.items && copy.count == 0 && + copy.cap == 0 && ok; + } + ok = py_facts_equal(&r->py_namespace, &original) && ok; + cbm_arena_destroy(&arena); + } + /* Rebuilding into an existing result must replace old complete facts on + * failure; do not leave a stale successful manifest attached. */ + if (r) { + CBMExtractCtx ctx = {.arena = &r->arena, + .result = r, + .source = source, + .source_len = (int)strlen(source), + .language = CBM_LANG_PYTHON}; + cbm_py_namespace_test_capture_fail_after(0); + CBMPyNamespaceStatus status = cbm_extract_python_namespace_facts(&ctx); + cbm_py_namespace_test_capture_fail_after(-1); + ok = status == CBM_PY_NS_INCOMPLETE && !r->py_namespace.items && + r->py_namespace.count == 0 && ok; + cbm_free_result(r); + } + cbm_arena_destroy(&reference_arena); + ASSERT_TRUE(ok); + ASSERT_TRUE(capture_complete); + ASSERT_TRUE(copy_complete); + ASSERT_GT(capture_failures, 2); + ASSERT_GT(copy_failures, 2); + PASS(); +} + +TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema) { + CBMPyNamespaceFacts zero = {0}; + bool ok = cbm_py_namespace_facts_valid(&zero) && zero.status == CBM_PY_NS_NOT_CAPTURED; + CBMPyNamespaceFact event = {.kind = CBM_PY_NS_ALL_SET, + .sequence_kind = CBM_PY_NS_SEQUENCE_LIST}; + CBMPyNamespaceFacts facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE, + .items = &event, + .count = 1, + .cap = 1}; + ok = cbm_py_namespace_facts_valid(&facts) && ok; + for (int variant = 0; variant < 8; variant++) { + CBMPyNamespaceFact bad_event = event; + CBMPyNamespaceFacts bad = facts; + bad.items = &bad_event; + if (variant == 0) + bad.version++; + if (variant == 1) + bad.cap = 0; + if (variant == 2) + bad.count = -1; + if (variant == 3) + bad_event.kind = (CBMPyNamespaceFactKind)999; + if (variant == 4) + bad_event.name_count = 1; + if (variant == 5) + bad_event.flags = UINT32_MAX; + if (variant == 6) + bad.language = CBM_LANG_JAVA; + if (variant == 7) + bad.status = CBM_PY_NS_INCOMPLETE; + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + ok = !cbm_py_namespace_facts_valid(&bad) && + !cbm_py_namespace_facts_copy(&arena, &bad, ©) && + copy.status == CBM_PY_NS_INCOMPLETE && copy.count == 0 && !copy.items && ok; + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload) { + char source[] = "from . import Thing as Local\n" + "__all__ = ['Local', '', 'a,b', '_private']\n" + "if enabled:\n Local = dynamic\n" + "__all__ = []\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena retained; + cbm_arena_init(&retained); + CBMPyNamespaceFacts expected = {0}; + bool ok = r && !r->has_error && !r->parse_incomplete && r->py_namespace.count == 4 && + cbm_py_namespace_facts_copy(&retained, &r->py_namespace, &expected); + memset(source, 'x', strlen(source)); /* facts cannot borrow the caller's input */ + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_facts_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 1) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + ok = spill && ok; + if (r && spill && ok) { + cbm_result_compact(r); + cbm_result_compact(r); + ok = r->arena.nblocks == 1 && r->py_namespace.cap == r->py_namespace.count && + py_facts_equal(&r->py_namespace, &expected) && + py_facts_owned(&r->arena, &r->py_namespace); + if (ok && cbm_result_spill_park(spill, 0, 0, r)) { + r = NULL; + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, 0, &header) && + header.py_namespace.status == CBM_PY_NS_COMPLETE && header.py_namespace.count == 4; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, 0); + ok = loaded && loaded->cached_tree == NULL && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && + py_facts_equal(&loaded->py_namespace, &expected) && ok; + if (loaded) + cbm_free_result(loaded); + } + } else { + ok = false; + } + } + if (r) + cbm_free_result(r); + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + /* Original result/source tree and both reloads are gone; the independent + * copied payload still owns the relative empty module and literal strings. */ + ok = py_facts_owned(&retained, &expected) && cbm_py_namespace_facts_valid(&expected) && + expected.count == 4 && expected.items[0].relative_level == 1 && + strcmp(expected.items[0].module_name, "") == 0 && expected.items[1].name_count == 4 && + strcmp(expected.items[1].names[2], "a,b") == 0 && ok; + cbm_arena_destroy(&retained); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unavailable_and_empty_survive_spill) { + CBMPyNamespaceFacts states[] = { + {0}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_ALLOCATION}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_JAVASCRIPT, + .status = CBM_PY_NS_NOT_APPLICABLE}, + }; + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_fact_states_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 4) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + bool ok = spill != NULL; + for (int i = 0; spill && i < 4; i++) { + CBMFileResult *r = cbm_result_alloc(); + if (!r) { + ok = false; + break; + } + cbm_arena_init(&r->arena); + r->module_qn = cbm_arena_strdup(&r->arena, "t.metadata"); + r->py_namespace = states[i]; + cbm_result_compact(r); + cbm_result_compact(r); + bool ready = + r->module_qn && r->arena.nblocks == 1 && py_facts_equal(&r->py_namespace, &states[i]); + if (!ready || !cbm_result_spill_park(spill, 0, i, r)) { + ok = false; + cbm_free_result(r); + continue; + } + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, i, &header) && + header.py_namespace.status == states[i].status && header.py_namespace.count == 0 && ok; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, i); + ok = loaded && py_facts_equal(&loaded->py_namespace, &states[i]) && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && ok; + if (loaded) + cbm_free_result(loaded); + } + } + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects) { + static const char source[] = "import alpha, beta.mod as Beta\n" + "from pkg import (One, Two as Local)\n" + "from pkg import later\n" + "def later():\n pass\n" + "from pkg import __all__\n" + "def __all__():\n pass\n" + "values = {obj: 1}\n" + "class HasDescriptor:\n field = descriptor\n" + "__all__ = ['escaped\\n']\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 14 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "later") && + has_def(r, "Class", "HasDescriptor"); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[0].local_name, "alpha") == 0 && + f[1].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[1].local_name, "Beta") == 0 && + strcmp(f[1].module_name, "beta.mod") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].member_name, "One") == 0 && f[3].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[3].member_name, "Two") == 0 && strcmp(f[3].local_name, "Local") == 0 && + f[4].kind == CBM_PY_NS_IMPORT_NAME && f[5].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[5].local_name, "later") == 0 && f[6].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[6].local_name, "__all__") == 0 && f[7].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[7].local_name, "__all__") == 0 && f[8].kind == CBM_PY_NS_UNKNOWN && + f[9].kind == CBM_PY_NS_SHADOW && strcmp(f[9].local_name, "values") == 0 && + f[10].kind == CBM_PY_NS_UNKNOWN && f[11].kind == CBM_PY_NS_SHADOW && + f[11].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && f[12].kind == CBM_PY_NS_UNKNOWN && + f[13].kind == CBM_PY_NS_ALL_UNKNOWN; + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -8974,6 +9453,16 @@ SUITE(extraction) { /* Initialize extraction library */ cbm_init(); + RUN_TEST(python_namespace_facts_preserve_order_and_import_provenance); + RUN_TEST(python_namespace_facts_distinguish_all_operations_and_unknowns); + RUN_TEST(python_namespace_facts_unknown_effects_and_body_stability); + RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); + RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); + RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); + RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); + RUN_TEST(python_namespace_facts_unavailable_and_empty_survive_spill); + RUN_TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects); + /* Wide-flat-file linearity (ms-typescript hang) */ RUN_TEST(extract_wide_flat_file_is_linear); #if defined(CBM_CALL_REFERENCE_LOOKUP_TEST_API) && CBM_CALL_REFERENCE_LOOKUP_TEST_API diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 633e47aacf..a7d38e79af 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8522,6 +8522,337 @@ static int python_module_callers_pipeline_case(bool parallel) { PASS(); } +/* Insert after python_module_exact_lsp_edge. Parent registers the four TESTs. */ +static const char namespace_admission_control[] = "def target():\n" + " return 1\n" + "def keep():\n" + " return target()\n"; + +static const char *const namespace_admission_env_keys[] = {"CBM_WORKERS", + "CBM_INDEX_SINGLE_THREAD"}; + +typedef struct { + char root[512]; + char db[768]; + char *saved[2]; + bool created; + bool saved_env; +} namespace_admission_fixture; + +typedef struct { + int rc; + int errors; + int format; + cbm_incremental_route_t route; + bool migrated; + bool healthy; + char project[512]; + char *json; +} namespace_admission_result; + +static bool namespace_admission_begin(namespace_admission_fixture *f) { + int n = snprintf(f->root, sizeof(f->root), "%s/cbm-namespace-admission-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(f->root) || !cbm_mkdtemp(f->root)) + return false; + f->created = true; + n = snprintf(f->db, sizeof(f->db), "%s/namespace.db", f->root); + if (n <= 0 || (size_t)n >= sizeof(f->db)) + return false; + for (int i = 0; i < 2; i++) { + const char *old = getenv(namespace_admission_env_keys[i]); + f->saved[i] = old ? strdup(old) : NULL; + if (old && !f->saved[i]) + return false; + } + f->saved_env = true; + int env_rc = cbm_setenv(namespace_admission_env_keys[0], "1", 1); + env_rc |= cbm_setenv(namespace_admission_env_keys[1], "1", 1); + return env_rc == 0 && + python_module_write_fixture_file(f->root, "control.py", namespace_admission_control); +} + +static bool namespace_admission_finish(namespace_admission_fixture *f) { + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + if (f->saved_env) + restore_rc |= f->saved[i] ? cbm_setenv(namespace_admission_env_keys[i], f->saved[i], 1) + : cbm_unsetenv(namespace_admission_env_keys[i]); + free(f->saved[i]); + } + cbm_pipeline_incremental_test_reset_faults(); + int cleanup_rc = f->created ? th_rmtree(f->root) : 0; + return restore_rc == 0 && cleanup_rc == 0; +} + +static bool namespace_admission_exact_hash(cbm_store_t *store, const char *project, + const char *path, const char *source) { + cbm_file_hash_t row = {0}; + char expected[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(source, strlen(source), expected); + bool correct = cbm_store_get_file_hash(store, project, path, &row) == CBM_STORE_OK && + row.sha256 && strcmp(row.sha256, expected) == 0; + cbm_store_clear_file_hash(&row); + return correct; +} + +static namespace_admission_result namespace_admission_run(namespace_admission_fixture *f, + const char *watch, const char *source, + int expected_files) { + namespace_admission_result result = { + .rc = -1, .errors = -1, .format = -1, .route = CBM_INCREMENTAL_ROUTE_NONE}; + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *pipeline = cbm_pipeline_new(f->root, f->db, CBM_MODE_FULL); + if (!pipeline) + return result; + result.rc = cbm_pipeline_run(pipeline); + result.route = cbm_pipeline_incremental_test_last_route(); + result.migrated = cbm_pipeline_had_format_migration(pipeline); + cbm_file_error_t *errors = NULL; + cbm_pipeline_get_file_errors(pipeline, &errors, &result.errors); + const char *project = cbm_pipeline_project_name(pipeline); + int n = project ? snprintf(result.project, sizeof(result.project), "%s", project) : -1; + cbm_store_t *store = result.rc == 0 && n > 0 && (size_t)n < sizeof(result.project) + ? cbm_store_open_path(f->db) + : NULL; + if (store) { + cbm_node_t *files = NULL; + int file_count = 0; + bool file_query = cbm_store_find_nodes_by_label(store, project, "File", &files, + &file_count) == CBM_STORE_OK; + int watched_files = 0; + for (int i = 0; file_query && i < file_count; i++) + if (files[i].file_path && strcmp(files[i].file_path, watch) == 0) + watched_files++; + cbm_store_free_nodes(files, file_count); + bool positive = python_module_exact_lsp_edge(store, project, "control.keep", "Function", + "control.target", "Function"); + bool hashes = namespace_admission_exact_hash(store, project, "control.py", + namespace_admission_control) && + namespace_admission_exact_hash(store, project, watch, source); + bool format_ok = cbm_store_get_format_version(store, &result.format) == CBM_STORE_OK; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + bool row_query = + cbm_store_get_lsp_surfaces(store, project, &rows, &row_count) == CBM_STORE_OK; + int matches = 0; + for (int i = 0; row_query && i < row_count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, watch) != 0) + continue; + matches++; + if (rows[i].defs_json && !result.json) + result.json = strdup(rows[i].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + bool python_row = matches == 1 && result.json && + cbm_lsp_surface_python_change(result.json, result.json) == 0 && + cbm_lsp_surface_python_change(result.json, NULL) == 1; + result.healthy = result.errors == 0 && file_query && file_count == expected_files && + watched_files == 1 && positive && hashes && format_ok && + result.format == CBM_INDEX_FORMAT_VERSION && row_query && python_row; + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + return result; +} + +TEST(pipeline_python_namespace_admission_upgrade_then_noop) { + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result first = {0}, upgraded = {0}, unchanged = {0}; + bool stamped = false; + if (setup) { + first = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + cbm_store_t *store = first.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + int old_format = -1; + stamped = cbm_store_set_format_version(store, 1) == CBM_STORE_OK && + cbm_store_get_format_version(store, &old_format) == CBM_STORE_OK && + old_format == 1; + cbm_store_close(store); + } + if (stamped) + upgraded = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + if (upgraded.healthy) + unchanged = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + } + bool stable = first.json && upgraded.json && unchanged.json && + cbm_lsp_surface_python_change(first.json, upgraded.json) == 0 && + cbm_lsp_surface_python_change(upgraded.json, unchanged.json) == 0; + free(first.json); + free(upgraded.json); + free(unchanged.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && stamped && cleaned); + ASSERT_TRUE(first.healthy && upgraded.healthy && unchanged.healthy && stable); + ASSERT_TRUE(upgraded.migrated); + ASSERT_FALSE(unchanged.migrated); + ASSERT_EQ(unchanged.route, CBM_INCREMENTAL_ROUTE_NOOP); + PASS(); +} + +TEST(pipeline_python_namespace_admission_body_alias_and_all) { + static const char initial[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 1\n"; + static const char body_edit[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char alias_edit[] = "from math import ceil as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char all_edit[] = "from math import ceil as Alias\n" + "__all__ = []\n" + "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result phases[4] = {0}; + const char *sources[] = {initial, body_edit, alias_edit, all_edit}; + bool wrote_all = setup; + for (int i = 0; i < 4 && wrote_all; i++) { + wrote_all = python_module_write_fixture_file(fixture.root, "subject.py", sources[i]); + if (wrote_all) + phases[i] = namespace_admission_run(&fixture, "subject.py", sources[i], 2); + if (!phases[i].healthy) + break; + } + bool all_healthy = + phases[0].healthy && phases[1].healthy && phases[2].healthy && phases[3].healthy; + bool comparisons = all_healthy && + cbm_lsp_surface_python_change(phases[0].json, phases[1].json) == 0 && + cbm_lsp_surface_python_change(phases[1].json, phases[2].json) == 1 && + cbm_lsp_surface_python_change(phases[2].json, phases[3].json) == 1; + for (int i = 0; i < 4; i++) + free(phases[i].json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && wrote_all && cleaned && all_healthy && comparisons); + ASSERT_EQ(phases[1].route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(phases[2].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(phases[3].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + +TEST(pipeline_python_namespace_admission_deleted_and_empty) { + const char *sources[] = {"def removed():\n return 3\n", "# empty namespace\n"}; + bool setups[2] = {false, false}, removed[2] = {false, false}; + bool healthy[2] = {false, false}, absent[2] = {false, false}, cleaned[2] = {false, false}; + cbm_incremental_route_t routes[2] = {CBM_INCREMENTAL_ROUTE_NONE, CBM_INCREMENTAL_ROUTE_NONE}; + for (int i = 0; i < 2; i++) { + namespace_admission_fixture fixture = {0}; + setups[i] = namespace_admission_begin(&fixture); + if (setups[i]) + setups[i] = python_module_write_fixture_file(fixture.root, "gone.py", sources[i]); + namespace_admission_result before = {0}, after = {0}; + if (setups[i]) + before = namespace_admission_run(&fixture, "gone.py", sources[i], 2); + if (before.healthy) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/gone.py", fixture.root); + removed[i] = n > 0 && (size_t)n < sizeof(path) && cbm_unlink(path) == 0; + } + if (removed[i]) + after = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + healthy[i] = before.healthy && after.healthy; + routes[i] = after.route; + cbm_store_t *store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_node_t *nodes = NULL; + int count = 0; + absent[i] = cbm_store_find_nodes_by_file(store, after.project, "gone.py", &nodes, + &count) == CBM_STORE_OK && + count == 0; + cbm_store_free_nodes(nodes, count); + cbm_store_close(store); + } + free(before.json); + free(after.json); + cleaned[i] = namespace_admission_finish(&fixture); + } + for (int i = 0; i < 2; i++) { + ASSERT_TRUE(setups[i] && removed[i] && healthy[i] && absent[i] && cleaned[i]); + ASSERT_EQ(routes[i], CBM_INCREMENTAL_ROUTE_FORCED_FULL); + } + PASS(); +} + +TEST(pipeline_python_namespace_admission_invalid_equal_hash) { + static const char initial[] = "def body():\n return 1\n"; + static const char edited[] = "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + if (setup) + setup = python_module_write_fixture_file(fixture.root, "subject.py", initial); + namespace_admission_result before = {0}, after = {0}; + bool corrupted = false, wrote = false; + char *old_sha = NULL, *new_sha = NULL; + if (setup) + before = namespace_admission_run(&fixture, "subject.py", initial, 2); + cbm_store_t *store = before.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, before.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + old_sha = rows[i].surface_sha ? strdup(rows[i].surface_sha) : NULL; + cbm_lsp_surface_row_t invalid = rows[i]; + invalid.defs_json = "{}"; /* Leave the old hash intact: validate before equality. */ + corrupted = old_sha && cbm_lsp_surface_python_change("{}", "{}") == -1 && + cbm_store_upsert_lsp_surface_batch(store, &invalid, 1) == CBM_STORE_OK; + break; + } + cbm_store_free_lsp_surfaces(rows, count); + rows = NULL; + count = 0; + bool confirmed = false; + rc = corrupted ? cbm_store_get_lsp_surfaces(store, before.project, &rows, &count) + : CBM_STORE_ERR; + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + confirmed = rows[i].defs_json && strcmp(rows[i].defs_json, "{}") == 0 && + rows[i].surface_sha && strcmp(rows[i].surface_sha, old_sha) == 0; + break; + } + corrupted = corrupted && confirmed; + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + if (corrupted) + wrote = python_module_write_fixture_file(fixture.root, "subject.py", edited); + if (wrote) + after = namespace_admission_run(&fixture, "subject.py", edited, 2); + store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, after.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (rows[i].rel_path && strcmp(rows[i].rel_path, "subject.py") == 0 && + rows[i].surface_sha) { + new_sha = strdup(rows[i].surface_sha); + break; + } + } + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + bool equal_hash = old_sha && new_sha && strcmp(old_sha, new_sha) == 0; + bool equal_valid_surface = before.healthy && after.healthy && + cbm_lsp_surface_python_change(before.json, after.json) == 0; + free(old_sha); + free(new_sha); + free(before.json); + free(after.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && corrupted && wrote && cleaned); + ASSERT_TRUE(before.healthy && after.healthy && equal_hash && equal_valid_surface); + ASSERT_EQ(after.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + TEST(pipeline_python_module_callers_sequential) { return python_module_callers_pipeline_case(false); } @@ -16243,6 +16574,247 @@ TEST(pipeline_seq_ts_cross_uses_shared_registry) { PASS(); } +/* Build a real extraction surface without publishing a database. The returned + * JSON outlives both the extraction and collector arenas. */ +static char *pipeline_namespace_surface(const char *source, bool include_defs) { + CBMFileResult *fr = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "namespace_probe", "pkg/__init__.py", 0, NULL, NULL); + if (!fr || fr->has_error || fr->parse_incomplete) { + cbm_free_result(fr); + return NULL; + } + CBMArena arena; + cbm_arena_init(&arena); + CBMFileResult *cache[] = {fr}; + cbm_file_info_t files[] = {{.rel_path = "pkg/__init__.py", .language = CBM_LANG_PYTHON}}; + char *modules[] = {NULL}; + int starts[2] = {0, 0}; + int def_count = 0; + CBMLSPDef *defs = include_defs + ? cbm_pxc_collect_all_defs(NULL, &arena, cache, files, 1, + "namespace_probe", modules, &def_count, starts) + : NULL; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + char *json = NULL; + if (cbm_lsp_surface_build_rows(NULL, "namespace_probe", cache, files, 1, defs, starts, &rows, + &row_count) == 0 && + row_count == 1 && rows[0].defs_json) { + json = strdup(rows[0].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + free(defs); + free(modules[0]); + cbm_arena_destroy(&arena); + cbm_free_result(fr); + return json; +} + +TEST(pipeline_python_namespace_change_admission) { + const char *sources[] = { + "def target(value):\n return 1\n", + "def target(value):\n return 2\n", + "def target(value, extra):\n return 1\n", + "from first import Target as Alias\n", + "from second import Target as Alias\n", + "from first import Target as Alias\nfrom second import Target as Alias\n", + "from second import Target as Alias\nfrom first import Target as Alias\n", + "__all__ = []\n", + "__all__ = ['Alias']\n", + "__all__ = dynamic_names()\n", + "# empty namespace\n", + }; + enum { count = (int)(sizeof(sources) / sizeof(sources[0])) }; + char *json[count]; + bool complete = true; + for (int i = 0; i < count; i++) { + json[i] = pipeline_namespace_surface(sources[i], true); + complete = complete && json[i] != NULL; + } + int body = -9, signature = -9, alias = -9, order = -9; + int empty = -9, literal = -9, unknown = -9, deletion = -9, transition = -9; + const char *non_python = "{\"v\":2,\"lang\":0,\"lsp\":[],\"reg\":[],\"py\":null}"; + if (complete) { + body = cbm_lsp_surface_python_change(json[0], json[1]); + signature = cbm_lsp_surface_python_change(json[0], json[2]); + alias = cbm_lsp_surface_python_change(json[3], json[4]); + order = cbm_lsp_surface_python_change(json[5], json[6]); + empty = cbm_lsp_surface_python_change(json[10], json[7]); + literal = cbm_lsp_surface_python_change(json[7], json[8]); + unknown = cbm_lsp_surface_python_change(json[7], json[9]); + deletion = cbm_lsp_surface_python_change(json[10], NULL); + transition = cbm_lsp_surface_python_change(non_python, json[10]); + } + for (int i = 0; i < count; i++) { + free(json[i]); + } + ASSERT_TRUE(complete); + ASSERT_EQ(body, 0); + ASSERT_EQ(signature, 1); + ASSERT_EQ(alias, 1); + ASSERT_EQ(order, 1); + ASSERT_EQ(empty, 1); + ASSERT_EQ(literal, 1); + ASSERT_EQ(unknown, 1); + ASSERT_EQ(deletion, 1); + ASSERT_EQ(transition, 1); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, NULL), 0); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, non_python), 0); + PASS(); +} + +static char *pipeline_namespace_payload_json(const char *json) { + yyjson_doc *doc = json ? yyjson_read(json, strlen(json), 0) : NULL; + if (!doc) { + return NULL; + } + yyjson_val *payload = yyjson_obj_get(yyjson_doc_get_root(doc), "py"); + char *copy = payload ? yyjson_val_write(payload, 0, NULL) : NULL; + yyjson_doc_free(doc); + return copy; +} + +TEST(pipeline_python_namespace_codec_owns_ordered_facts) { + const char *source = "def plain(value):\n return value\n" + "from .one import First as Alias\n" + "from .two import Second as Alias\n" + "__all__ = ['Alias', '_private', '', 'odd|name']\n" + "__all__ += ('tail',)\n" + "__all__ = []\n" + "__all__ = dynamic_names()\n"; + char *full = pipeline_namespace_surface(source, true); + char *probe = pipeline_namespace_surface(source, false); + char *full_facts = pipeline_namespace_payload_json(full); + char *probe_facts = pipeline_namespace_payload_json(probe); + bool parity = full_facts && probe_facts && strcmp(full_facts, probe_facts) == 0; + bool distinct_defs = full && probe && strcmp(full, probe) != 0; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t copied = {0}, failed = {0}; + cbm_py_namespace_test_copy_fail_after(0); + int failed_rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &failed); + cbm_py_namespace_test_copy_fail_after(-1); + int rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &copied); + free(full); + free(probe); + free(full_facts); + free(probe_facts); + + /* The original result, collector, JSON documents and serialized bytes are + * all gone before the copied strings and nested arrays are inspected. */ + bool origin = rc == 1 && copied.module_qn && copied.rel_path && + strcmp(copied.module_qn, "namespace_probe.pkg.__init__") == 0 && + strcmp(copied.rel_path, "pkg/__init__.py") == 0; + bool valid = rc == 1 && cbm_py_namespace_facts_valid(&copied.facts) && + copied.facts.status == CBM_PY_NS_COMPLETE && + copied.facts.language == CBM_LANG_PYTHON; + int imports = 0; + bool import_order = true, names = false, append = false, empty = false, unknown = false; + if (valid) { + for (int i = 0; i < copied.facts.count; i++) { + const CBMPyNamespaceFact *f = &copied.facts.items[i]; + if (f->kind == CBM_PY_NS_IMPORT_NAME) { + const char *module = imports == 0 ? "one" : "two"; + const char *member = imports == 0 ? "First" : "Second"; + import_order = import_order && imports < 2 && f->relative_level == 1 && + f->module_name && strcmp(f->module_name, module) == 0 && + f->member_name && strcmp(f->member_name, member) == 0 && + f->local_name && strcmp(f->local_name, "Alias") == 0; + imports++; + } + if (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 4) { + names = f->sequence_kind == CBM_PY_NS_SEQUENCE_LIST && + strcmp(f->names[0], "Alias") == 0 && strcmp(f->names[1], "_private") == 0 && + strcmp(f->names[2], "") == 0 && strcmp(f->names[3], "odd|name") == 0; + } + if (f->kind == CBM_PY_NS_ALL_APPEND) { + append = f->sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && f->name_count == 1 && + strcmp(f->names[0], "tail") == 0; + } + empty = + empty || (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 0 && f->names == NULL); + unknown = unknown || f->kind == CBM_PY_NS_ALL_UNKNOWN; + } + } + bool failure_empty = failed.facts.items == NULL && failed.facts.count == 0 && + failed.module_qn == NULL && failed.rel_path == NULL; + cbm_arena_destroy(&arena); + ASSERT_TRUE(parity); + ASSERT_TRUE(distinct_defs); + ASSERT_EQ(failed_rc, -1); + ASSERT_TRUE(failure_empty); + ASSERT_EQ(rc, 1); + ASSERT_TRUE(origin); + ASSERT_TRUE(valid); + ASSERT_EQ(imports, 2); + ASSERT_TRUE(import_order); + ASSERT_TRUE(names); + ASSERT_TRUE(append); + ASSERT_TRUE(empty); + ASSERT_TRUE(unknown); + PASS(); +} + +TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) { + /* Numeric values below are the frozen version-1 namespace wire values. + * Even an empty LSP array must validate its namespace envelope. */ + const char *bad[] = { + "{\"v\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1.0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"v\":2,\"lang\":0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":3,\"failure\":2,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":99,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\\u0000suffix\",\"status\":2," + "\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":999,\"d\":0,\"s\":0,\"r\":0,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":8,\"d\":0,\"s\":1,\"r\":0,\"f\":0,\"l\":4294967296," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + }; + bool rejected = true; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + CBMLSPDef *defs = NULL; + int ns_rc = cbm_lsp_surface_python_namespace_from_json(&arena, bad[i], &facts); + int defs_rc = cbm_lsp_surface_defs_from_json(&arena, bad[i], &defs); + int compare_rc = cbm_lsp_surface_python_change(bad[i], bad[i]); + int delete_rc = cbm_lsp_surface_python_change(bad[i], NULL); + rejected = rejected && ns_rc == -1 && defs_rc == -1 && compare_rc == -1 && + delete_rc == -1 && facts.facts.items == NULL && defs == NULL; + cbm_arena_destroy(&arena); + } + const char *unknown = + "{\"v\":2,\"lang\":1,\"lsp\":[],\"reg\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":7,\"d\":0,\"s\":0,\"r\":7,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}"; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + int unknown_rc = cbm_lsp_surface_python_namespace_from_json(&arena, unknown, &facts); + int equal_rc = cbm_lsp_surface_python_change(unknown, unknown); + bool retained = unknown_rc == 1 && facts.facts.count == 1 && + facts.facts.items[0].kind == CBM_PY_NS_UNKNOWN && + facts.facts.items[0].reason == CBM_PY_NS_REASON_PARSE; + cbm_arena_destroy(&arena); + ASSERT_TRUE(rejected); + ASSERT_EQ(unknown_rc, 1); + ASSERT_EQ(equal_rc, 0); + ASSERT_TRUE(retained); + PASS(); +} + /* Object arrays are non-flat in yyjson. Exercise ordered decoding and arena * ownership across a large surface, including nested string arrays. */ TEST(pipeline_lsp_surface_large_object_array_decode) { @@ -16250,7 +16822,7 @@ TEST(pipeline_lsp_surface_large_object_array_decode) { const size_t capacity = (size_t)count * 192 + 64; char *json = malloc(capacity); ASSERT_NOT_NULL(json); - size_t used = (size_t)snprintf(json, capacity, "{\"v\":1,\"lsp\":["); + size_t used = (size_t)snprintf(json, capacity, "{\"v\":2,\"lang\":0,\"py\":null,\"lsp\":["); for (int i = 0; i < count; i++) { int n = snprintf(json + used, capacity - used, "%s{\"qn\":\"pkg.f%d\",\"sn\":\"f%d\",\"lb\":\"Function\"," @@ -16342,7 +16914,7 @@ TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant) { snprintf(sha_baseline, sizeof(sha_baseline), "%s", row_a->surface_sha); /* The row is the versioned codec envelope with the def present. */ snprintf(json_probe, sizeof(json_probe), "%.20s", row_a->defs_json); - ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":1") != NULL); + ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":2") != NULL); ASSERT_TRUE(strstr(row_a->defs_json, "surface_probe") != NULL); } else if (round == 1) { ASSERT_STR_EQ(row_a->surface_sha, sha_baseline); @@ -17302,6 +17874,13 @@ SUITE(pipeline) { RUN_TEST(pipeline_fixture_file_parent_is_preserved); RUN_TEST(pipeline_fixture_node_count_requires_exact_source_nodes); RUN_TEST(pipeline_nested_fixture_graph_has_endpoints); + RUN_TEST(pipeline_python_namespace_admission_upgrade_then_noop); + RUN_TEST(pipeline_python_namespace_admission_body_alias_and_all); + RUN_TEST(pipeline_python_namespace_admission_deleted_and_empty); + RUN_TEST(pipeline_python_namespace_admission_invalid_equal_hash); + RUN_TEST(pipeline_python_namespace_change_admission); + RUN_TEST(pipeline_python_namespace_codec_owns_ordered_facts); + RUN_TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows); RUN_TEST(pipeline_lsp_surface_large_object_array_decode); RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */ From fc05929d3648f7540bcfd6c0be5fd20dcda72563 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 22:39:02 +0200 Subject: [PATCH 8/9] perf(python): index unresolved submodule prefix probes Reuse a stable arena-owned function QN order for ambiguity marking and submodule prefix lookup, preserving exact lookup policy and appended tails. Add prefix/index failure regressions and a pipeline operation-count case. Signed-off-by: Martin Vogel --- internal/cbm/lsp/py_lsp.c | 100 ++++---- internal/cbm/lsp/py_lsp.h | 8 + internal/cbm/lsp/type_registry.c | 93 +++++++ internal/cbm/lsp/type_registry.h | 32 +++ scripts/memory-core-baseline.txt | 1 - tests/test_complexity.c | 182 +++++++++++++- tests/test_py_lsp.c | 405 +++++++++++++++++++++++++++++++ 7 files changed, 759 insertions(+), 62 deletions(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 0c8571bc2d..a36b27835f 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -36,6 +36,11 @@ #define PY_LSP_MAX_EVAL_DEPTH 256 #include "lsp_work.h" +#ifdef CBM_ENABLE_TEST_SEAMS +_Atomic uint64_t cbm_py_submodule_probes = 0; +_Atomic uint64_t cbm_py_submodule_probe_visits = 0; +#endif + #ifdef CBM_ENABLE_TEST_SEAMS static _Thread_local int py_test_memo_allocations_left = -1; static _Thread_local bool py_test_depth_failure; @@ -163,51 +168,32 @@ static bool py_func_is_exact_callable_value(const CBMRegisteredFunc *func) { (!func->decorator_qns || !func->decorator_qns[0]); } -static int py_func_qn_pointer_cmp(const void *left, const void *right) { - const CBMRegisteredFunc *a = *(CBMRegisteredFunc *const *)left; - const CBMRegisteredFunc *b = *(CBMRegisteredFunc *const *)right; - const char *aqn = a ? a->qualified_name : NULL; - const char *bqn = b ? b->qualified_name : NULL; - if (!aqn || !bqn) - return aqn ? 1 : bqn ? -1 : 0; - return strcmp(aqn, bqn); -} - -/* The graph identifies functions by QN. When Python source redefines the - * same QN, the runtime's last binding is knowable but the graph no longer has - * one occurrence-exact definition target. Mark the whole duplicate group so - * callable values fail closed without changing ordinary call overload/rebind - * behavior. */ -static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) { - if (!registry || registry->func_count <= 1) +/* Reuse the QN snapshot for duplicate marks and later submodule probes. + * The graph cannot identify one exact callable value for a duplicate QN; + * ordinary exact lookups retain their original registration-order policy. */ +static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry, CBMArena *idx_arena) { + if (!registry) + return; + cbm_registry_build_func_qn_order(registry, idx_arena); + if (registry->func_count <= 1) return; - size_t count = (size_t)registry->func_count; - CBMRegisteredFunc **sorted = count <= SIZE_MAX / sizeof(*sorted) - ? (CBMRegisteredFunc **)malloc(count * sizeof(*sorted)) - : NULL; - if (!sorted) { - /* Allocation failure must reduce precision, never fabricate it. */ + const CBMRegistryQNEntry *sorted = registry->func_qn_sorted; + if (!sorted || registry->func_qn_sorted_upto != registry->func_count) { for (int i = 0; i < registry->func_count; i++) registry->funcs[i].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; return; } - for (int i = 0; i < registry->func_count; i++) - sorted[i] = ®istry->funcs[i]; - qsort(sorted, count, sizeof(*sorted), py_func_qn_pointer_cmp); - for (size_t first = 0; first < count;) { - size_t end = first + 1; - const char *qn = sorted[first]->qualified_name; - while (qn && end < count && sorted[end]->qualified_name && - strcmp(qn, sorted[end]->qualified_name) == 0) { + int count = registry->func_qn_sorted_count; + for (int first = 0; first < count;) { + int end = first + 1; + while (end < count && strcmp(sorted[first].qualified_name, sorted[end].qualified_name) == 0) end++; - } - if (qn && end - first > 1) { - for (size_t i = first; i < end; i++) - sorted[i]->flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; + if (end - first > 1) { + for (int i = first; i < end; i++) + registry->funcs[sorted[i].func_index].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; } first = end; } - free(sorted); } /* Depth-guarded entry for the AST call-resolution walk. The walk recurses once @@ -1584,6 +1570,22 @@ static const CBMType *py_iterable_element_type(PyLSPContext *ctx, const CBMType /* The real recursive-descent evaluator. Never call directly — go through * the memoizing, depth-guarded py_eval_expr_type wrapper below * (every recursive call inside this body already does). */ +/* Existing function-only submodule predicate, after exact symbol/type misses. */ +static bool py_names_submodule(PyLSPContext *ctx, const char *qn) { + const char *prefix = qn ? cbm_arena_sprintf(ctx->arena, "%s.", qn) : NULL; + if (!prefix) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + uint64_t visits = 0; + bool found = cbm_registry_has_func_qn_prefix(ctx->registry, prefix, &visits); + atomic_fetch_add_explicit(&cbm_py_submodule_probes, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_submodule_probe_visits, visits, memory_order_relaxed); + return found; +#else + return cbm_registry_has_func_qn_prefix(ctx->registry, prefix, NULL); +#endif +} + static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) return cbm_type_unknown(); @@ -1704,25 +1706,7 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, qn); if (rt) return cbm_type_named(ctx->arena, qn); - // Submodule: if any registered function/type has qn starting - // with ".." then mod.attr is itself a module. - // Linear scan over registry funcs is O(R) per access; we - // skip it for the common case where mod.attr is already - // matched as a function/type above. With ~900 stdlib funcs - // and many module-attr accesses per file, this can dominate - // — keeping the loop tight and bailing early on first match. - const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn); - size_t prefix_len = strlen(prefix); - bool is_submodule = false; - CBMFreeFuncIter all_funcs; - cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); - for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { - const char *fqn = all_funcs.reg->funcs[i].qualified_name; - if (fqn && strncmp(fqn, prefix, prefix_len) == 0) { - is_submodule = true; - } - } - if (is_submodule) + if (py_names_submodule(ctx, qn)) return cbm_type_module(ctx->arena, qn); return cbm_type_unknown(); } @@ -5343,7 +5327,6 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, for (int i = 0; i < result->defs.count; i++) { py_register_def(arena, ®, &result->defs.items[i], module_qn); } - py_mark_ambiguous_callable_bindings(®); /* Hash-index the registry before the walk (#1527). Unfinalized, every * lookup scanned the stdlib plus all of this file's definitions, which @@ -5354,6 +5337,7 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, * scratch arena that dies with this call, not in the result arena. */ CBMArena idx_arena; cbm_arena_init(&idx_arena); + py_mark_ambiguous_callable_bindings(®, &idx_arena); reg.index_first_registered = true; cbm_registry_finalize_into(®, &idx_arena); @@ -5604,7 +5588,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in ctx.eval_failure = CBM_LSP_SCOPE_FAILED; goto cross_cleanup; } - py_mark_ambiguous_callable_bindings(®); + py_mark_ambiguous_callable_bindings(®, &idx_arena); /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" * comment for the rationale. */ @@ -5648,7 +5632,7 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i return NULL; /* caller owns the shared arena and other registries */ } - py_mark_ambiguous_callable_bindings(reg); + py_mark_ambiguous_callable_bindings(reg, arena); cbm_registry_finalize(reg); reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ return reg; diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index b20e733279..9532f3adbc 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -238,4 +238,12 @@ typedef struct { bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out); +#ifdef CBM_ENABLE_TEST_SEAMS +#include +/* Process-wide comparison counts for deterministic resolver operation tests. + * Only missing exact symbol/type attributes reach this submodule predicate. */ +extern _Atomic uint64_t cbm_py_submodule_probes; +extern _Atomic uint64_t cbm_py_submodule_probe_visits; +#endif + #endif // CBM_LSP_PY_LSP_H diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 216c347794..d2b1f5cb11 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -985,6 +985,99 @@ const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, return cbm_registry_lookup_func(reg, buf); } +/* A stable total order, independent of qsort's treatment of equal keys. */ +static int registry_qn_entry_cmp(const void *left, const void *right) { + const CBMRegistryQNEntry *a = left; + const CBMRegistryQNEntry *b = right; + int cmp = strcmp(a->qualified_name, b->qualified_name); + return cmp ? cmp : (a->func_index > b->func_index) - (a->func_index < b->func_index); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int registry_qn_order_allocations_left = -1; +void cbm_registry_test_qn_order_fail_after(int successful_allocations) { + registry_qn_order_allocations_left = successful_allocations; +} +#endif + +void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena) { + if (!reg || !arena || reg->read_only) + return; + /* A failed rebuild must not leave an old snapshot authoritative. */ + reg->func_qn_sorted = NULL; + reg->func_qn_sorted_count = 0; + reg->func_qn_sorted_upto = 0; + if (reg->func_count <= 0 || (size_t)reg->func_count > SIZE_MAX / sizeof(CBMRegistryQNEntry)) + return; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_qn_order_allocations_left == 0) + return; + if (registry_qn_order_allocations_left > 0) + registry_qn_order_allocations_left--; +#endif + CBMRegistryQNEntry *sorted = cbm_arena_alloc(arena, (size_t)reg->func_count * sizeof(*sorted)); + if (!sorted) + return; + int count = 0; + for (int i = 0; i < reg->func_count; i++) { + if (reg->funcs[i].qualified_name) { + sorted[count].qualified_name = reg->funcs[i].qualified_name; + sorted[count].func_index = i; + count++; + } + } + qsort(sorted, (size_t)count, sizeof(*sorted), registry_qn_entry_cmp); + reg->func_qn_sorted = sorted; + reg->func_qn_sorted_count = count; + reg->func_qn_sorted_upto = reg->func_count; +} + +static bool registry_link_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + size_t length, uint64_t *visits) { + int scan_from = 0; + if (reg->func_qn_sorted && reg->func_qn_sorted_upto <= reg->func_count) { + int lo = 0, hi = reg->func_qn_sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (visits) + (*visits)++; + if (strcmp(reg->func_qn_sorted[mid].qualified_name, prefix) < 0) + lo = mid + 1; + else + hi = mid; + } + if (lo < reg->func_qn_sorted_count) { + if (visits) + (*visits)++; + if (strncmp(reg->func_qn_sorted[lo].qualified_name, prefix, length) == 0) + return true; + } + scan_from = reg->func_qn_sorted_upto; + } + for (int i = scan_from; i < reg->func_count; i++) { + if (visits) + (*visits)++; + const char *qn = reg->funcs[i].qualified_name; + if (qn && strncmp(qn, prefix, length) == 0) + return true; + } + return false; +} + +bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + uint64_t *visits) { + if (!prefix) + return false; + size_t length = strlen(prefix); + /* The old iterator omits shadowed fallback QNs, whose equal head QN has + * the same prefix answer. This boolean union needs no winner selection. */ + for (const CBMTypeRegistry *link = reg; link; link = link->fallback) { + if (registry_link_has_func_qn_prefix(link, prefix, length, visits)) + return true; + } + return false; +} + // Count parameters in a FUNC signature. static int count_func_params(const CBMRegisteredFunc *f) { if (!f || !f->signature || f->signature->kind != CBM_TYPE_FUNC) diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index 1ee284bf7c..b8f01009b9 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -74,6 +74,12 @@ typedef struct { int slot; // bucket slot this entry sits in (for resize) } CBMRegistryHashEntry; +/* A borrowed immutable QN and stable array index, never a movable func pointer. */ +typedef struct { + const char *qualified_name; + int func_index; +} CBMRegistryQNEntry; + // Cross-file type/function registry. typedef struct CBMTypeRegistry { CBMRegisteredFunc *funcs; @@ -150,6 +156,13 @@ typedef struct CBMTypeRegistry { // hashed chains answer with the LAST registration (the Tier-2 behavior, // left unchanged). bool index_first_registered; + + /* Additional prefix/duplicate index; exact lookup order is unchanged. + * Covers non-NULL QNs in funcs[0..func_qn_sorted_upto), ordered by (QN,index). + * Appended functions remain visible through a tail scan. */ + CBMRegistryQNEntry *func_qn_sorted; + int func_qn_sorted_count; + int func_qn_sorted_upto; } CBMTypeRegistry; // Initialize a registry. @@ -198,6 +211,25 @@ const CBMRegisteredFunc *cbm_registry_lookup_func(const CBMTypeRegistry *reg, const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, const char *package_qn, const char *name); +/* Build in an arena that outlives every use (owner arena for shared registries, + * walk scratch for per-file registries). QN strings and registered identities + * must remain immutable while a snapshot is used; end its use before changing + * identities, then rebuild before querying again. Appends do not invalidate it. + * Never frees caller arenas. Sealed registries are unchanged. A failed rebuild + * discards the old order and queries scan the complete registry instead. */ +void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena); + +/* Boolean starts-with over all function/method QNs in the fallback chain. + * Does not allocate or mutate. NULL prefix is false; empty prefix matches any + * non-NULL QN. Optional visits accumulates compared entries, including tail + * scans; fully indexed links use a lower bound plus at most one candidate. */ +bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + uint64_t *visits); +#ifdef CBM_ENABLE_TEST_SEAMS +/* -1 disables; zero fails this allocation until reset; positive counts down. */ +void cbm_registry_test_qn_order_fail_after(int successful_allocations); +#endif + // Resolve type alias chain: follow alias_of until concrete type found (max 16 levels). const CBMRegisteredType *cbm_registry_resolve_alias(const CBMTypeRegistry *reg, const char *type_qn); diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 138ef0f77d..2321c430e3 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -10,7 +10,6 @@ internal/cbm/extract_defs.c 4 internal/cbm/helpers.c 3 internal/cbm/lsp/c_lsp.c 4 internal/cbm/lsp/perl_lsp.c 21 -internal/cbm/lsp/py_lsp.c 2 internal/cbm/macro_table.c 1 internal/cbm/service_patterns.c 2 internal/cbm/sqlite_writer.c 74 diff --git a/tests/test_complexity.c b/tests/test_complexity.c index f20eac16ed..57f7321768 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -50,11 +50,13 @@ #include "cbm.h" #include "discover/discover.h" #include "lang_specs.h" +#include "lsp/py_lsp.h" #include "tree_sitter/api.h" #include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "store/store.h" +#include "yyjson/yyjson.h" #include #include @@ -86,6 +88,9 @@ enum { * code path. The sequential path's own per-file registry work is bounded * by the 50-file ceiling and is deliberately out of scope here. */ CX_BIGPKG_FILES_PER_K = 20, + CX_PYEXT_FILES_PER_MOD = 20, + CX_PYEXT_DEFS_PER_FILE = 120, + CX_PYEXT_PROBES_PER_FILE = 3, }; /* Linear growth bounds for a 2x input doubling. Fixed structural overhead @@ -306,6 +311,44 @@ static int cx_build_bigpkg(const char *root, int k) { return 0; } +/* Every file probes absent external submodules and retains a real local-call + * control. Independent module copies grow the shared registry without adding + * cross-copy dependencies. Both sizes exceed the parallel-driver threshold. */ +static int cx_build_pyext(const char *root, int k) { + char dir[1024]; + char path[1200]; + for (int m = 0; m < k; m++) { + int n = snprintf(dir, sizeof(dir), "%s/pyext/mod%d", root, m); + if (n < 0 || (size_t)n >= sizeof(dir) || th_mkdir_p(dir) != 0) + return -1; + for (int j = 0; j < CX_PYEXT_FILES_PER_MOD; j++) { + n = snprintf(path, sizeof(path), "%s/p%d.py", dir, j); + if (n < 0 || (size_t)n >= sizeof(path)) + return -1; + FILE *f = fopen(path, "w"); + if (!f) + return -1; + bool written = fprintf(f, "import extlib\n\n") >= 0; + for (int d = 0; written && d < CX_PYEXT_DEFS_PER_FILE; d++) { + written = + fprintf(f, "def cx_pyext_definition_%d(x):\n return x + %d\n\n", d, d) >= 0; + } + if (written) + written = + fprintf(f, "def cx_pyext_use():\n return (cx_pyext_definition_0(0)") >= 0; + for (int probe = 0; written && probe < CX_PYEXT_PROBES_PER_FILE; probe++) { + written = fprintf(f, " + extlib.sub%d.helper(%d)", probe, probe) >= 0; + } + if (written) + written = fprintf(f, ")\n") >= 0; + int closed = fclose(f); + if (!written || closed != 0) + return -1; + } + } + return 0; +} + /* ── Metrics ─────────────────────────────────────────────────────────── */ /* Per-pass timing capture: a TEE log sink parses `pass.timing` lines during a @@ -357,16 +400,87 @@ typedef struct { uint64_t fallback_rows; uint64_t imp_nodes; /* symbols scored by the importance pass */ uint64_t imp_name_visits; /* same-name-group members visited by that pass */ + uint64_t py_probes; + uint64_t py_probe_visits; + int pyext_files; + int pyext_defs; + int pyext_controls; + int errors; double wall_s; } CxMetrics; +/* Count the intended corpus, then prove each use() kept its exact local LSP + * call. Query errors are failures, never indistinguishable from zero rows. */ +static bool cx_pyext_control(cbm_store_t *store, const char *project, const cbm_node_t *caller) { + if (!caller->qualified_name || !caller->file_path) + return false; + const char *dot = strrchr(caller->qualified_name, '.'); + if (!dot || dot - caller->qualified_name > 1000) + return false; + char target_qn[1200]; + int n = snprintf(target_qn, sizeof(target_qn), "%.*s.cx_pyext_definition_0", + (int)(dot - caller->qualified_name), caller->qualified_name); + if (n < 0 || (size_t)n >= sizeof(target_qn)) + return false; + cbm_node_t target = {0}; + cbm_edge_t *edges = NULL; + int count = 0; + bool found = false; + if (cbm_store_find_node_by_qn(store, project, target_qn, &target) == CBM_STORE_OK && + target.label && strcmp(target.label, "Function") == 0 && target.file_path && + strcmp(target.file_path, caller->file_path) == 0 && + cbm_store_find_edges_by_source_type(store, caller->id, "CALLS", &edges, &count) == + CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + if (edges[i].target_id != target.id || !edges[i].properties_json) + continue; + yyjson_doc *doc = + yyjson_read(edges[i].properties_json, strlen(edges[i].properties_json), 0); + const char *source = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "source")) : NULL; + found = found || (source && strcmp(source, "lsp") == 0); + yyjson_doc_free(doc); + } + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&target); + return found; +} + +static bool cx_pyext_population(cbm_store_t *store, const char *project, CxMetrics *out) { + cbm_node_t *nodes = NULL; + int count = 0; + bool ok = cbm_store_find_nodes_by_label(store, project, "File", &nodes, &count) == CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + if (nodes[i].file_path && strncmp(nodes[i].file_path, "pyext/", 6) == 0) + out->pyext_files++; + } + cbm_store_free_nodes(nodes, count); + nodes = NULL; + count = 0; + ok = ok && + cbm_store_find_nodes_by_label(store, project, "Function", &nodes, &count) == CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + const cbm_node_t *node = &nodes[i]; + if (!node->file_path || strncmp(node->file_path, "pyext/", 6) != 0 || !node->name) + continue; + static const char definition_prefix[] = "cx_pyext_definition_"; + if (strncmp(node->name, definition_prefix, sizeof(definition_prefix) - 1) == 0) + out->pyext_defs++; + if (strcmp(node->name, "cx_pyext_use") == 0 && cx_pyext_control(store, project, node)) + out->pyext_controls++; + } + cbm_store_free_nodes(nodes, count); + return ok; +} + static double cx_now_s(void) { struct timespec ts; cbm_profile_now(&ts); return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; } -static int cx_run(const char *root, const char *db_path, CxMetrics *out) { +static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool pyext) { memset(out, 0, sizeof(*out)); uint64_t d0; @@ -379,6 +493,8 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows(); uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed); uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); + uint64_t pp0 = atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + uint64_t pv0 = atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -390,6 +506,7 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { } int rc = cbm_pipeline_run(p); out->wall_s = cx_now_s() - t0; + cbm_pipeline_get_file_errors(p, NULL, &out->errors); cbm_log_set_sink(NULL); int captured = atomic_load_explicit(&g_cx_pass_count, memory_order_relaxed); out->pass_count = captured < CX_MAX_PASSES ? captured : CX_MAX_PASSES; @@ -417,6 +534,10 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->imp_name_visits = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0; + out->py_probes = atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) - pp0; + out->py_probe_visits = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) - pv0; + cbm_store_t *s = cbm_store_open_path(db_path); if (!s) { return -1; @@ -427,8 +548,9 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->lang_nodes[t] = cbm_store_count_nodes_scoped(s, project, CX_TEMPLATES[t].dirname); out->lang_edges[t] = cbm_store_count_edges_scoped(s, project, CX_TEMPLATES[t].dirname); } + bool extra_ok = !pyext || cx_pyext_population(s, project, out); cbm_store_close(s); - return 0; + return extra_ok ? 0 : -1; } static double cx_ratio(double num, double den) { @@ -463,7 +585,8 @@ static int cx_measure(CxCorpusBuilder build, const char *prefix_a, const char *p char db_b[600]; snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a); snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b); - if (cx_run(root_a, db_a, out_a) == 0 && cx_run(root_b, db_b, out_b) == 0) { + bool pyext = build == cx_build_pyext; + if (cx_run(root_a, db_a, out_a, pyext) == 0 && cx_run(root_b, db_b, out_b, pyext) == 0) { rc = 0; } } @@ -616,6 +739,58 @@ TEST(complexity_shared_package_growth_stays_linear) { PASS(); } +TEST(complexity_py_submodule_probe_avoids_registry_scan) { + static const char *const keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {NULL, NULL}; + bool ready = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + if (old && !saved[i]) + ready = false; + } + CxMetrics a = {0}, b = {0}; + int rc = -1, restored = 0; + if (ready) { + int configured = cbm_setenv(keys[0], "2", 1); + configured |= cbm_unsetenv(keys[1]); + if (configured == 0) + rc = cx_measure(cx_build_pyext, "cbm_cxpy_a", "cbm_cxpy_b", &a, &b); + for (int i = 0; i < 2; i++) + restored |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + } + for (int i = 0; i < 2; i++) + free(saved[i]); + /* cx_measure removes both owned corpora before any assertion below. */ + ASSERT_TRUE(ready); + ASSERT_EQ(restored, 0); + ASSERT_EQ(rc, 0); + int files = CX_K_BASE * CX_PYEXT_FILES_PER_MOD; + ASSERT_EQ(a.errors, 0); + ASSERT_EQ(b.errors, 0); + ASSERT_EQ(a.pyext_files, files); + ASSERT_EQ(b.pyext_files, files * 2); + ASSERT_EQ(a.pyext_defs, files * CX_PYEXT_DEFS_PER_FILE); + ASSERT_EQ(b.pyext_defs, files * 2 * CX_PYEXT_DEFS_PER_FILE); + ASSERT_EQ(a.pyext_controls, files); + ASSERT_EQ(b.pyext_controls, files * 2); + ASSERT_GT((long long)a.py_probes, (long long)CX_MIN_BASE_WORK); + ASSERT_GT((long long)b.py_probes, (long long)CX_MIN_BASE_WORK); + ASSERT_GT((long long)a.py_probe_visits, 0); + ASSERT_GT((long long)b.py_probe_visits, 0); + double probes_ratio = cx_ratio((double)b.py_probes, (double)a.py_probes); + double visits_ratio = cx_ratio((double)b.py_probe_visits, (double)a.py_probe_visits); + printf(" py_probes %llu -> %llu (%.2f), visits %llu -> %llu (%.2f)\n", + (unsigned long long)a.py_probes, (unsigned long long)b.py_probes, probes_ratio, + (unsigned long long)a.py_probe_visits, (unsigned long long)b.py_probe_visits, + visits_ratio); + ASSERT_TRUE(probes_ratio >= CX_RATIO_LO && probes_ratio <= CX_RATIO_HI); + ASSERT_TRUE(visits_ratio <= CX_RATIO_HI); + /* The direct registry test also enforces a logarithmic comparison bound, + * independently of fixed stdlib size and this corpus-growth ratio. */ + PASS(); +} + /* Throughput report — information only, never a gate (CI-determinism rule: * rates depend on the machine and scheduler, so a threshold would be a * lottery). Written locally under private/ (gitignored); CBM_COMPLEXITY_ @@ -847,5 +1022,6 @@ SUITE(complexity) { RUN_TEST(complexity_perfile_registry_work_is_linear); RUN_TEST(complexity_importance_scoring_is_linear); RUN_TEST(complexity_shared_package_growth_stays_linear); + RUN_TEST(complexity_py_submodule_probe_avoids_registry_scan); RUN_TEST(complexity_throughput_report_written); } diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 8ffe4540d9..628d2de6b4 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -3768,7 +3768,412 @@ TEST(pylsp_class_frame_failure_rolls_back_raw_and_cross_outputs) { PASS(); } +/* The prefix index is an existence oracle, never an exact-symbol selector. */ +static bool pylsp_prefix_scan(const CBMTypeRegistry *reg, const char *prefix) { + if (!reg || !prefix) + return false; + CBMFreeFuncIter iter; + cbm_registry_all_funcs_chain(reg, &iter); + size_t length = strlen(prefix); + for (int i; (i = cbm_free_func_iter_next(&iter)) >= 0;) { + const char *qn = iter.reg->funcs[i].qualified_name; + if (qn && strncmp(qn, prefix, length) == 0) + return true; + } + return false; +} + +static bool pylsp_prefix_add(CBMTypeRegistry *reg, const char *qn, const char *receiver) { + int before = reg->func_count; + cbm_registry_add_func( + reg, + (CBMRegisteredFunc){.qualified_name = qn, .short_name = "call", .receiver_type = receiver}); + return reg->func_count == before + 1; +} + +static bool pylsp_prefix_oracles(const CBMTypeRegistry *reg) { + const char *prefixes[] = {NULL, "", "pkg.", "pkg.sub.", + "pkg.sub.call", "pkg.su.", "pkg.submarine.", "pkg.Type.", + "only.tail.", "missing.", "zzzz."}; + for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) { + uint64_t visits = 17; + bool indexed = cbm_registry_has_func_qn_prefix(reg, prefixes[i], &visits); + if (indexed != pylsp_prefix_scan(reg, prefixes[i]) || visits < 17 || + cbm_registry_has_func_qn_prefix(reg, prefixes[i], NULL) != indexed || + (!prefixes[i] && visits != 17)) + return false; + } + return true; +} + +TEST(pylsp_prefix_order_matches_iterator_and_chain_shadowing) { + CBMArena arena, scratch; + cbm_arena_init(&arena); + cbm_arena_init(&scratch); + CBMTypeRegistry head, middle, tail, empty; + cbm_registry_init(&head, &arena); + cbm_registry_init(&middle, &arena); + cbm_registry_init(&tail, &arena); + cbm_registry_init(&empty, &arena); + bool ok = pylsp_prefix_add(&head, NULL, NULL) && pylsp_prefix_add(&head, "", NULL) && + pylsp_prefix_add(&head, "pkg.sub.call", NULL) && + pylsp_prefix_add(&head, "pkg.sub.call", NULL) && + pylsp_prefix_add(&head, "pkg.Type.call", "pkg.Type") && + pylsp_prefix_add(&middle, "pkg.sub.call", "different.receiver") && + pylsp_prefix_add(&middle, "pkg.submarine.call", NULL) && + pylsp_prefix_add(&tail, "only.tail.call", NULL); + head.fallback = &middle; + middle.fallback = &tail; + ok = pylsp_prefix_oracles(NULL) && pylsp_prefix_oracles(&empty) && + pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&head, &scratch); + ok = head.func_qn_sorted && head.func_qn_sorted_count == 4 && head.func_qn_sorted_upto == 5 && + pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&tail, &arena); + ok = tail.func_qn_sorted && pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&middle, &scratch); + ok = middle.func_qn_sorted && pylsp_prefix_oracles(&head) && ok; + if (head.func_qn_sorted) { + int duplicate_count = 0, previous = -1; + for (int i = 0; i < head.func_qn_sorted_count; i++) { + const CBMRegistryQNEntry *entry = &head.func_qn_sorted[i]; + ok = entry->func_index >= 0 && entry->func_index < head.func_count && + entry->qualified_name == head.funcs[entry->func_index].qualified_name && ok; + if (strcmp(entry->qualified_name, "pkg.sub.call") == 0) { + ok = entry->func_index > previous && ok; + previous = entry->func_index; + duplicate_count++; + } + } + ok = duplicate_count == 2 && ok; + } + size_t owned_before = cbm_arena_total(&arena), scratch_before = cbm_arena_total(&scratch); + ok = pylsp_prefix_oracles(&head) && cbm_arena_total(&arena) == owned_before && + cbm_arena_total(&scratch) == scratch_before && ok; + /* An indexed fallback also works behind an unindexed empty head. */ + empty.fallback = &head; + ok = pylsp_prefix_oracles(&empty) && ok; + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_survives_array_growth_rebuild_and_seal) { + CBMArena arena, scratch; + cbm_arena_init(&arena); + cbm_arena_init(&scratch); + CBMTypeRegistry reg, unindexed; + cbm_registry_init(®, &arena); + cbm_registry_init(&unindexed, &arena); + bool ok = pylsp_prefix_add(®, "pkg.old.call", NULL); + cbm_registry_build_func_qn_order(®, &scratch); + CBMRegisteredFunc *original_array = reg.funcs; + const CBMRegistryQNEntry *order = reg.func_qn_sorted; + for (int i = 0; i < 140; i++) { + const char *qn = cbm_arena_sprintf(&arena, "tail.mod%03d.call", i); + ok = qn && pylsp_prefix_add(®, qn, NULL) && ok; + } + ok = reg.func_count == 141 && reg.funcs != original_array && reg.func_qn_sorted == order && + reg.func_qn_sorted_upto == 1 && + cbm_registry_has_func_qn_prefix(®, "tail.mod139.", NULL) && + !cbm_registry_has_func_qn_prefix(®, "tail.mod140.", NULL) && ok; + cbm_registry_build_func_qn_order(®, &scratch); + ok = reg.func_qn_sorted && reg.func_qn_sorted_upto == reg.func_count && + cbm_registry_has_func_qn_prefix(®, "tail.mod139.", NULL) && ok; + /* End use of the old snapshot before replacing an identity. No lookup is + * permitted until the complete order has been rebuilt. */ + if (reg.func_count == 141) { + reg.funcs[0].qualified_name = "pkg.renamed.call"; + cbm_registry_build_func_qn_order(®, &scratch); + ok = !cbm_registry_has_func_qn_prefix(®, "pkg.old.", NULL) && + cbm_registry_has_func_qn_prefix(®, "pkg.renamed.", NULL) && ok; + } + order = reg.func_qn_sorted; + reg.read_only = true; + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &scratch); + cbm_registry_test_qn_order_fail_after(-1); + cbm_registry_add_func(®, (CBMRegisteredFunc){.qualified_name = "forbidden.call"}); + ok = reg.func_qn_sorted == order && reg.func_count == 141 && + cbm_registry_has_func_qn_prefix(®, "pkg.renamed.", NULL) && + !cbm_registry_has_func_qn_prefix(®, "forbidden.", NULL) && ok; + ok = pylsp_prefix_add(&unindexed, "sealed.call", NULL) && ok; + unindexed.read_only = true; + cbm_registry_build_func_qn_order(&unindexed, &scratch); + uint64_t visits = 0; + ok = !unindexed.func_qn_sorted && + cbm_registry_has_func_qn_prefix(&unindexed, "sealed.", &visits) && visits == 1 && ok; + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_allocation_failure_preserves_oracle) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = pylsp_prefix_add(®, "pkg.sub.call", NULL) && pylsp_prefix_add(®, NULL, NULL) && + pylsp_prefix_add(®, "only.tail.call", NULL); + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + uint64_t visits = 0; + ok = !reg.func_qn_sorted && reg.func_qn_sorted_count == 0 && reg.func_qn_sorted_upto == 0 && + pylsp_prefix_oracles(®) && !cbm_registry_has_func_qn_prefix(®, "zzzz.", &visits) && + visits == 3 && ok; + cbm_registry_test_qn_order_fail_after(1); + cbm_registry_build_func_qn_order(®, &arena); + bool first_built = reg.func_qn_sorted && reg.func_qn_sorted_upto == reg.func_count; + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + ok = first_built && !reg.func_qn_sorted && reg.func_qn_sorted_count == 0 && + reg.func_qn_sorted_upto == 0 && pylsp_prefix_oracles(®) && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.func_qn_sorted && reg.func_qn_sorted_count == 2 && reg.func_qn_sorted_upto == 3 && + pylsp_prefix_oracles(®) && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_has_absolute_comparison_bound) { + enum { FUNCTIONS = 1024, PROBES = 32, BOUND = 12 }; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = true; + for (int i = FUNCTIONS - 1; i >= 0; i--) { + const char *qn = cbm_arena_sprintf(&arena, "project.mod%04d.call", i); + ok = qn && pylsp_prefix_add(®, qn, NULL) && ok; + } + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.func_count == FUNCTIONS && reg.func_qn_sorted_count == FUNCTIONS && + reg.func_qn_sorted_upto == FUNCTIONS && reg.func_qn_sorted && ok; + for (int i = 0; i < PROBES; i++) { + char hit[64], miss[64]; + int a = snprintf(hit, sizeof(hit), "project.mod%04d.", i * 31); + int b = snprintf(miss, sizeof(miss), "project.mod%04dx.", i * 31); + uint64_t positive = 0, negative = 0; + bool ready = a > 0 && (size_t)a < sizeof(hit) && b > 0 && (size_t)b < sizeof(miss); + ok = ready && cbm_registry_has_func_qn_prefix(®, hit, &positive) && + !cbm_registry_has_func_qn_prefix(®, miss, &negative) && positive > 0 && + positive <= BOUND && negative > 0 && negative <= BOUND && + pylsp_prefix_scan(®, hit) && !pylsp_prefix_scan(®, miss) && ok; + } + /* Failure restores the reference scan, which must inspect every entry on + * this absent suffix. This is a deterministic contrast, not a timing ratio. */ + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + uint64_t visits = 0; + ok = !reg.func_qn_sorted && !cbm_registry_has_func_qn_prefix(®, "zzzz.", &visits) && + visits == FUNCTIONS && visits > BOUND && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_preserves_exact_selection_and_ambiguity) { + CBMArena arena; + cbm_arena_init(&arena); + bool ok = true; + for (int first = 0; first < 2; first++) { + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + ok = pylsp_prefix_add(®, "pkg.duplicate", NULL) && + pylsp_prefix_add(®, "pkg.duplicate", NULL) && ok; + const CBMRegisteredFunc *linear = cbm_registry_lookup_func(®, "pkg.duplicate"); + ok = reg.func_count == 2 && linear == ®.funcs[0] && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = cbm_registry_lookup_func(®, "pkg.duplicate") == linear && ok; + reg.index_first_registered = first != 0; + cbm_registry_finalize(®); + const CBMRegisteredFunc *selected = cbm_registry_lookup_func(®, "pkg.duplicate"); + ok = reg.func_count == 2 && selected == ®.funcs[first ? 0 : 1] && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = cbm_registry_lookup_func(®, "pkg.duplicate") == selected && ok; + } + CBMLSPDef defs[] = { + {.qualified_name = "pkg.duplicate", + .short_name = "duplicate", + .label = "Function", + .def_module_qn = "pkg", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.duplicate", + .short_name = "duplicate", + .label = "Function", + .def_module_qn = "pkg", + .return_types = "str", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.unique", + .short_name = "unique", + .label = "Function", + .def_module_qn = "pkg", + .lang = CBM_LANG_PYTHON}, + }; + for (int fail = 0; fail < 2; fail++) { + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 3); + cbm_registry_test_qn_order_fail_after(-1); + ok = reg && reg->read_only && reg->func_count > 3 && ok; + if (reg) { + int duplicate_count = 0, unique_count = 0; + for (int i = 0; i < reg->func_count; i++) { + const CBMRegisteredFunc *f = ®->funcs[i]; + bool ambiguous = (f->flags & CBM_FUNC_FLAG_AMBIGUOUS_BINDING) != 0; + if (fail) + ok = ambiguous && ok; + if (f->qualified_name && strcmp(f->qualified_name, "pkg.duplicate") == 0) { + duplicate_count++; + ok = ambiguous && ok; + } + if (f->qualified_name && strcmp(f->qualified_name, "pkg.unique") == 0) { + unique_count++; + ok = ambiguous == (fail != 0) && ok; + } + } + ok = duplicate_count == 2 && unique_count == 1 && + (fail ? reg->func_qn_sorted == NULL : reg->func_qn_sorted != NULL) && + cbm_registry_has_func_qn_prefix(reg, "pkg.", NULL) && + !cbm_registry_has_func_qn_prefix(reg, "pkg.missing.", NULL) && ok; + } + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +/* A negative must have one real extracted source occurrence and no accepted + * resolution; a missing carrier cannot make it pass. */ +static bool pylsp_prefix_site_unresolved(const CBMCallArray *calls, + const CBMResolvedCallArray *resolved, const char *source, + const char *site_text, const char *caller) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source), end = start + (uint32_t)strlen(site_text); + const CBMCall *found = NULL; + int count = 0; + for (int i = 0; i < calls->count; i++) { + const CBMCall *call = &calls->items[i]; + if (call->enclosing_func_qn && strcmp(call->enclosing_func_qn, caller) == 0 && + call->site_start_byte == start && call->site_end_byte == end) { + found = call; + count++; + } + } + return count == 1 && cbm_pipeline_find_lsp_resolution(resolved, found, false) == NULL; +} + +TEST(pylsp_prefix_raw_per_file_and_shared_keep_exact_calls) { + static const char source[] = "import pkg\nimport vendor\n" + "def ping():\n return 1\n" + "def run():\n" + " pkg.sub.ping()\n" + " pkg.su.ping()\n" + " vendor.external.ping()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "pkg.sub.ping", + .short_name = "ping", + .label = "Function", + .def_module_qn = "pkg.sub", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.submarine.ping", + .short_name = "ping", + .label = "Function", + .def_module_qn = "pkg.submarine", + .return_types = "str", + .lang = CBM_LANG_PYTHON}, + }; + const char *names[] = {"pkg", "vendor"}; + const char *qns[] = {"pkg", "vendor"}; + bool ok = true; + for (int fail = 0; fail < 2; fail++) { + uint64_t probes_before = + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + uint64_t visits_before = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "pkg", + "sub.py", 0, NULL, NULL); + cbm_registry_test_qn_order_fail_after(-1); + bool healthy = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + pylsp_module_has_def(r, "pkg.sub.ping") && + pylsp_module_has_def(r, "pkg.sub.run"); + ok = healthy && + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) > probes_before && + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) > + visits_before && + ok; + if (healthy) { + ok = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "pkg.sub.ping()", + "pkg.sub.run", "pkg.sub.ping") && + pylsp_prefix_site_unresolved(&r->calls, &r->resolved_calls, source, + "pkg.su.ping()", "pkg.sub.run") && + pylsp_prefix_site_unresolved(&r->calls, &r->resolved_calls, source, + "vendor.external.ping()", "pkg.sub.run") && + ok; + for (int shared_mode = 0; shared_mode < 2; shared_mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *reg = NULL; + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + if (shared_mode) + reg = cbm_py_build_cross_registry(&shared, defs, 2); + CBMResolvedCallArray out = {0}; + CBMLSPStatus status = CBM_LSP_SCOPE_FAILED; + probes_before = + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + visits_before = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + if (!shared_mode || reg) { + status = shared_mode + ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), "pkg.sub", reg, names, + qns, 2, r->cached_tree, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + "pkg.sub", defs, 2, names, qns, 2, + r->cached_tree, &out, NULL); + } + cbm_registry_test_qn_order_fail_after(-1); + ok = status == CBM_LSP_COMPLETE && + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) > + probes_before && + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) > + visits_before && + pylsp_module_site_joins(&r->calls, &out, source, "pkg.sub.ping()", + "pkg.sub.run", "pkg.sub.ping") && + pylsp_prefix_site_unresolved(&r->calls, &out, source, "pkg.su.ping()", + "pkg.sub.run") && + pylsp_prefix_site_unresolved(&r->calls, &out, source, "vendor.external.ping()", + "pkg.sub.run") && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + } + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(ok); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_prefix_order_matches_iterator_and_chain_shadowing); + RUN_TEST(pylsp_prefix_order_survives_array_growth_rebuild_and_seal); + RUN_TEST(pylsp_prefix_order_allocation_failure_preserves_oracle); + RUN_TEST(pylsp_prefix_order_has_absolute_comparison_bound); + RUN_TEST(pylsp_prefix_order_preserves_exact_selection_and_ambiguity); + RUN_TEST(pylsp_prefix_raw_per_file_and_shared_keep_exact_calls); + RUN_TEST(pylsp_classbody_calls_join_extracted_caller); RUN_TEST(pylsp_classbody_names_visible_to_later_statements_only); RUN_TEST(pylsp_decorator_and_base_calls_join_extracted_caller); From 6561da09190fb8877901a4344948f17655e98d04 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 23:14:30 +0200 Subject: [PATCH 9/9] fix(pipeline): preserve allocation policy in resolver cleanup Use tracked zeroed storage for the parallel LSP failure array and release it with the matching memory class after worker joins. Share import-filter failure cleanup without changing fallback state. Keep existing raw-allocation allowances unchanged. Signed-off-by: Martin Vogel --- src/pipeline/pass_lsp_cross.c | 9 +++++---- src/pipeline/pass_parallel.c | 9 +++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 643d3ff191..3ae60db0e5 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -2213,8 +2213,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); for (int i = 0; i < imp_count; i++) { if (!pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total)) { - free(selected); - return NULL; /* out_success remains false: dispatcher uses all_defs */ + goto failed; /* out_success remains false: dispatcher uses all_defs */ } } if (pxc_is_jvm_lang(caller_lang) && idx->namespace_ht) { @@ -2231,8 +2230,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef)); if (!out) { - free(selected); - return NULL; + goto failed; } int n = 0; @@ -2245,4 +2243,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef *out_success = true; free(selected); return out; +failed: + free(selected); + return NULL; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index af415e13b3..265017c507 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3877,6 +3877,10 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return 0; } + if (file_count < 0 || (size_t)file_count > SIZE_MAX / sizeof(const char *)) { + return CBM_NOT_FOUND; + } + cbm_log_info("parallel.resolve.start", "files", itoa_log(file_count), "workers", itoa_log(worker_count)); @@ -3886,7 +3890,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return CBM_NOT_FOUND; } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); - const char **lsp_failures = calloc((size_t)file_count, sizeof(*lsp_failures)); + const char **lsp_failures = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)file_count * sizeof(*lsp_failures)); if (!lsp_failures) { cbm_aligned_free(workers); return CBM_NOT_FOUND; @@ -3954,7 +3959,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], "lsp_skipped"); } - free(lsp_failures); + cbm_free(CBM_MEM_CLASS_OTHER, lsp_failures); /* Workers joined: the shared Rust registry (if built) is no longer read. * Free its dedicated arena + the lock (registry was self-contained: it strdup'd * all QNs, so freeing all_defs afterward is safe). */