diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cdee413b..2488063ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In JavaScript and TypeScript, a call on one of the browser's own objects, like `window.open('/help')`, `localStorage.getItem('token')`, `location.reload()`, `history.back()`, `window.localStorage.getItem('token')` or a worker's `self.postMessage(…)`, no longer links to a method of your own that only shares its name, like a `WindowService.open` or a `StorageService.getItem` an app wraps the browser in, so `codegraph callers` and impact on such a wrapper no longer list every direct browser call in the app. A name the file declares, imports or takes as a parameter, like `const history = createRouter()`, still calls what it holds, `this` still means the class or object it is written in, and so does a `self` the file sets to it, like `var self = this`. Re-index JavaScript and TypeScript projects after upgrading. Thanks @Gobakabana for the report and @KissFeng. (#2907) - In JavaScript and TypeScript, a method called on a freshly constructed object, like `new RegExp(p).exec(s)`, `new URL(u).toString()` or `new Runner().run()`, now links only to that class's own or inherited method, on the class the file imports when several share its name, and no longer to an unrelated project method of the same name; a class from outside your project links nothing. Re-index JavaScript and TypeScript projects after upgrading. Thanks @danusha2345. (#1945) - In Python, `import a.b as X` written inside a function now links its calls like `X.f()`, as the same import at module level already did. Thanks @Egrn for the report. (#2600) +- In Python, a call through a name the file imports as a module, like `other.update(…)` after `from pkg import other`, now links to that module only where nothing closer gives the name another value: a parameter, a local, a `with … as` or `except … as` target, a loop or comprehension variable, a lambda parameter or a `case` capture named `other` is that value, so its calls no longer show up as calls into the module. A `from pkg import third as other` written inside a function now links that function's `other.get()` to `third`, and no longer the rest of the file's. Re-index Python projects after upgrading. Thanks @BGamboa13. (#2632) - In Python, `from app import email` no longer links the importing file to a project variable named `app`, such as a FastAPI or Flask `app = FastAPI()`. (#2584) - In Python, a from-import of a module whose name has no dot, like `from loader import Registry` with `loader.py` at the repository root, or `from mypackage import helper` in a `src/` layout, now links the importing file to that module's file, as `import loader` and `from pkg.loader import Registry` already did. Before, the file's dependency on the module itself was left out, so `codegraph affected` and impact could miss the files that import it. When the file also imported something else under the module's name, like `from django.core import mail` beside `from mail import …`, the module was linked to that instead. Re-index Python projects after upgrading. - In Python, a call through a module-level instance, like `_svc.delete(obj)` after `_svc = SceneService()`, now links to the method its class has or inherits, even when other classes have a method of that name. (#2582) diff --git a/__tests__/python-receiver-rebinding.test.ts b/__tests__/python-receiver-rebinding.test.ts new file mode 100644 index 000000000..248510546 --- /dev/null +++ b/__tests__/python-receiver-rebinding.test.ts @@ -0,0 +1,235 @@ +/** + * A Python call through a name the file imports as a module reads that module + * only where nothing nearer binds the name: + * + * from pkg import other + * + * def real(): + * other.update({}) # pkg/other.py's update + * + * def by_param(other): + * other.update({}) # dict.update, or whatever the caller passed + * + * The import resolver read the file's imports alone, so a parameter, a local, + * a `with … as` / `except … as` / loop target, a walrus, a `case` capture or a + * lambda or comprehension variable named like the module still linked the + * call to the module's function. A function-local `from pkg import third as + * other` linked to `pkg.other` too, and so did, from every other function of + * the file, though it binds the name only where it is written. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +const FILES: Record = { + 'pyproject.toml': '[project]\nname = "pkg"\n', + 'src/pkg/__init__.py': '', + 'src/pkg/other.py': [ + 'def update(x=None):', + ' return x', + '', + '', + 'def persist(x=None):', + ' return x', + '', + '', + 'def get(key=None):', + ' return key', + '', + ].join('\n'), + 'src/pkg/third.py': ['def get(key=None):', ' return key', ''].join('\n'), + 'tests/shadow.py': [ + 'from pkg import other', + '', + '', + 'def real():', + ' other.update({"b": 2})', + ' return other.persist(1)', + '', + '', + 'def by_param(other):', + ' other.update({"a": 1})', + ' return other.persist(2)', + '', + '', + 'def by_with(path):', + ' with open(path, "w") as other:', + ' return other.persist(3)', + '', + '', + 'def by_assign():', + ' other = {}', + ' other.update({"c": 3})', + '', + '', + 'def by_for(items):', + ' for other in items:', + ' other.persist(4)', + '', + '', + 'def by_except():', + ' try:', + ' pass', + ' except OSError as other:', + ' return other.persist(5)', + '', + '', + 'def by_lambda():', + ' return lambda other: other.persist(6)', + '', + '', + 'def by_comprehension(items):', + ' return [other.persist(7) for other in items]', + '', + '', + 'def by_walrus(load):', + ' if (other := load()):', + ' return other.persist(8)', + '', + '', + 'def by_closure(other):', + ' def inner():', + ' return other.persist(9)', + ' return inner', + '', + '', + 'def by_capture(v):', + ' match v:', + ' case other:', + ' return other.persist(10)', + '', + '', + 'def by_local_import():', + ' from pkg import third as other', + ' return other.get(11)', + '', + '', + // Each of these reads the module the file imports. + 'def keyword_argument(run):', + ' run(', + ' other=1,', + ' )', + ' return other.persist(12)', + '', + '', + 'def in_a_string():', + ' print("with x as other", "other = 1") # other = 2', + ' return other.persist(13)', + '', + '', + 'def comprehension_of_others(items):', + ' return [x for x in other.persist(items)]', + '', + '', + 'def lambda_beside(run):', + ' return run(lambda other: 1, other.persist(14))', + '', + '', + 'def declared_global():', + ' global other', + ' return other.persist(15)', + '', + '', + 'def keyword_pattern(v):', + ' match v:', + ' case Point(other=1):', + ' return other.persist(16)', + '', + '', + 'def same_module_again():', + ' from pkg import other', + ' return other.get(17)', + '', + ].join('\n'), + // The module binds `other` with a plain import, which the file's mappings + // list after every from-import, the function-local one included. + 'tests/leak.py': [ + 'import pkg.other as other', + '', + '', + 'def local_third():', + ' from pkg import third as other', + ' return other.get(1)', + '', + '', + 'def untouched():', + ' return other.get(2)', + '', + ].join('\n'), +}; + +describe.each([ + ['default', 'LF'], + ['wasm', 'LF'], + ['default', 'CRLF'], + ['wasm', 'CRLF'], +])('a Python receiver the function binds itself is not the module the file imports (%s, %s)', (backend, eol) => { + let root = ''; + let cg: CodeGraph | undefined; + let kernel: string | undefined; + + beforeAll(async () => { + kernel = process.env.CODEGRAPH_KERNEL; + if (backend === 'wasm') process.env.CODEGRAPH_KERNEL = '0'; + else delete process.env.CODEGRAPH_KERNEL; + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-py-rebind-')); + for (const [rel, content] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), eol === 'CRLF' ? content.replace(/\n/g, '\r\n') : content); + } + cg = await CodeGraph.init(root, { index: true }); + }, 60_000); + + afterAll(() => { + cg?.destroy(); + if (root) fs.rmSync(root, { recursive: true, force: true }); + if (kernel === undefined) delete process.env.CODEGRAPH_KERNEL; + else process.env.CODEGRAPH_KERNEL = kernel; + }); + + /** `file:name` of each project function `name` in `file` calls, sorted. */ + const callsFrom = (file: string, name: string): string[] => { + const from = cg!.getNodesInFile(file).filter((n) => n.name === name && (n.kind === 'function' || n.kind === 'method')); + expect(from, `${name} in ${file}`).toHaveLength(1); + return cg!.getOutgoingEdgesFrom([from[0]!.id]) + .filter((e) => e.kind === 'calls') + .map((e) => cg!.getNode(e.target)!) + .filter((t) => t.filePath.startsWith('src/')) + .map((t) => `${t.filePath}:${t.name}`) + .sort(); + }; + + it('a call where nothing nearer binds the name reaches the module', () => { + expect(callsFrom('tests/shadow.py', 'real')).toEqual(['src/pkg/other.py:persist', 'src/pkg/other.py:update']); + }); + + it.each([ + 'by_param', 'by_with', 'by_assign', 'by_for', 'by_except', 'by_lambda', + 'by_comprehension', 'by_walrus', 'inner', 'by_capture', + ])('%s: a name the code around the call binds links no module function', (fn) => { + expect(callsFrom('tests/shadow.py', fn)).toEqual([]); + }); + + it('a function-local `from pkg import third as other` is that module', () => { + expect(callsFrom('tests/shadow.py', 'by_local_import')).toEqual(['src/pkg/third.py:get']); + expect(callsFrom('tests/leak.py', 'local_third')).toEqual(['src/pkg/third.py:get']); + }); + + it('a function-local import binds the name in that function only', () => { + expect(callsFrom('tests/leak.py', 'untouched')).toEqual(['src/pkg/other.py:get']); + }); + + it.each([ + ['keyword_argument', 'src/pkg/other.py:persist'], + ['in_a_string', 'src/pkg/other.py:persist'], + ['comprehension_of_others', 'src/pkg/other.py:persist'], + ['lambda_beside', 'src/pkg/other.py:persist'], + ['declared_global', 'src/pkg/other.py:persist'], + ['keyword_pattern', 'src/pkg/other.py:persist'], + ['same_module_again', 'src/pkg/other.py:get'], + ])('%s binds nothing of that name: the call reaches the module', (fn, target) => { + expect(callsFrom('tests/shadow.py', fn)).toEqual([target]); + }); +}); diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index eecd68068..e8743d0df 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -30,6 +30,8 @@ import { tsImportedTypeFile, reExportSource, pythonFunctionModuleAlias, + pythonReceiverBinding, + pythonModuleImportKeys, isPythonFromImportModule, tsGlobalScope, rustPathTarget, @@ -3127,7 +3129,7 @@ export function resolveViaImport( } // Use cached import mappings (avoids re-reading and re-parsing per ref) - const imports = context.getImportMappings(ref.filePath, ref.language); + let imports = context.getImportMappings(ref.filePath, ref.language); if (imports.length === 0 && !context.readFile(ref.filePath)) { return null; } @@ -3157,6 +3159,9 @@ export function resolveViaImport( // submodule (a file), not a symbol, so the generic symbol lookup below would // search the *package* for `certs` instead of looking inside the module. if (ref.language === 'python') { + // Only the imports the receiver means where it is read: none of the file's + // when the function binds the name itself (see pythonImportsInScope). + imports = pythonImportsInScope(ref, imports, context); const pyResult = resolvePythonModuleMember(ref, imports, context); if (pyResult) return pyResult; // Absolute dotted module import: `import conduit.apps.articles.signals` @@ -3373,6 +3378,53 @@ export function reachesAmbientModule(ref: UnresolvedRef, specifier: string, decl return LAST_NAME_KINDS.has(ref.referenceKind) && mappings.some((m) => leadsTo(m.source, () => false)); } +/** + * The file's import mappings as a Python dotted reference reads them. Its + * receiver is the module the file imports under that name only where nothing + * nearer binds the name: a parameter, a local, a `with … as` / `except … as` / + * loop target, a `case` capture, or a lambda or comprehension variable around + * the reference is that value instead, and none of the receiver's mappings + * applies there (`def by_param(other): other.update(…)` linked to the module + * imported as `other`). A function-local import of the name leaves only the + * mappings binding it the same way: the file's mappings read from-imports at + * any depth, so a local `from x import y as name` is listed beside the + * module-level import, and a local `import pkg.sub` binds the `pkg` that a + * module-level `import pkg` does. Other references and receivers keep every + * mapping. + */ +export function pythonImportsInScope(ref: UnresolvedRef, imports: ImportMapping[], context: ResolutionContext): ImportMapping[] { + if (ref.language !== 'python' || ref.referenceKind === 'imports') return imports; + const dot = ref.referenceName.indexOf('.'); + if (dot <= 0) return imports; + const receiver = ref.referenceName.slice(0, dot); + // `import pkg.mod` binds `pkg`, under a mapping named after `mod`. + const binds = (m: ImportMapping): boolean => m.localName === receiver || + (m.isNamespace && m.source.split('.')[0] === receiver && m.localName === m.source.split('.').pop()); + if (!imports.some(binds)) return imports; + const binding = pythonReceiverBinding(receiver, ref, context); + if (binding === 'module') { + // Where nothing nearer binds the name, the module's own import of it + // does — not a from-import written inside some other function, which the + // mappings list too (first, ahead of the module's `import pkg.other as + // other`). Asked only when the receiver's mappings disagree; with no + // module-level one found, every mapping stays as it was. + const keyOf = (m: ImportMapping): string => (m.isNamespace ? `${m.source}:*` : `${m.source}:${m.exportedName}`); + if (new Set(imports.filter(binds).map(keyOf)).size < 2) return imports; + const moduleLevel = pythonModuleImportKeys(receiver, ref.filePath, context); + const kept = imports.filter((m) => !binds(m) || m.isNamespace || moduleLevel.has(keyOf(m))); + return kept.some(binds) ? kept : imports; + } + return imports.filter((m) => { + if (!binds(m)) return true; + if (binding === 'local') return false; + // The module a namespace mapping binds the receiver to: `import x as + // receiver` / `import receiver` bind its source, `import receiver.sub` + // the top-level `receiver`. + if ('module' in binding) return m.isNamespace && (m.localName === receiver ? m.source : receiver) === binding.module; + return !m.isNamespace && m.source === binding.source && m.exportedName === binding.exportedName; + }); +} + /** * Resolve a Python qualified reference whose receiver is an imported MODULE: * `certs.where()` after `from . import certs`, `mod.func()` after `import mod` diff --git a/src/resolution/index.ts b/src/resolution/index.ts index 8bb90bdc5..cc7e04a3c 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -21,7 +21,7 @@ import { isImportableKind, CPP_DEFINE_SIGNATURE, } from './types'; -import { isPythonSelfCall, matchJsStoreBindingCall, isUnresolvedJsMemberCall, matchObjectPathCall, thisScopeCaller, isVisibleAcrossFiles, matchReference, matchByFilePath, matchFunctionRef, matchDottedCallChain, matchScopedCallChain, matchMethodCall, sameLanguageFamily, crossesCodeBoundary, gateLanguageMatch, dumpNameMatcherProfile, clearNameMatcherMemos, isRustNameInScope, isRustForeignImplCallable, isRustPathCallReachable, rustChainedCallByName, matchRustUseBoundName, CASE_INSENSITIVE_LANGUAGES, isDartMemberRead, matchDartMemberRead, isDartChainLink, matchDartChainLink, isDartAnnotation, matchDartAnnotation, isStdMethodName, isGoUnknownQualified, isGoProjectLocalCall, GO_STDLIB_PACKAGES, isGoBareName, goTypePositionTarget, goDotImportedSupertype, GO_TYPE_KINDS, matchGoAssertedCall, matchGoIndexedCall, matchGoChainedCall, preferGoBuildTwin, isGoAssertedLocal, isGoLocallyBound, isPythonFromImportModule, pythonAttrRefResolves, matchPythonTypedLocalCall, isImportShadowedByJsBinding, isManualMockPath, isTestSuitePath, matchNewReceiverCall, newReceiverClass, NEW_RECEIVER_SHAPE, isGuessedRecordAccessor, matchKotlinReceiverChain } from './name-matcher'; +import { isPythonSelfCall, matchJsStoreBindingCall, isUnresolvedJsMemberCall, matchObjectPathCall, thisScopeCaller, isVisibleAcrossFiles, matchReference, matchByFilePath, matchFunctionRef, matchDottedCallChain, matchScopedCallChain, matchMethodCall, sameLanguageFamily, crossesCodeBoundary, gateLanguageMatch, dumpNameMatcherProfile, clearNameMatcherMemos, isRustNameInScope, isRustForeignImplCallable, isRustPathCallReachable, rustChainedCallByName, matchRustUseBoundName, CASE_INSENSITIVE_LANGUAGES, isDartMemberRead, matchDartMemberRead, isDartChainLink, matchDartChainLink, isDartAnnotation, matchDartAnnotation, isStdMethodName, isGoUnknownQualified, isGoProjectLocalCall, GO_STDLIB_PACKAGES, isGoBareName, goTypePositionTarget, goDotImportedSupertype, GO_TYPE_KINDS, matchGoAssertedCall, matchGoIndexedCall, matchGoChainedCall, preferGoBuildTwin, isGoAssertedLocal, isGoLocallyBound, isPythonFromImportModule, pythonFunctionModuleAlias, pythonAttrRefResolves, matchPythonTypedLocalCall, isImportShadowedByJsBinding, isManualMockPath, isTestSuitePath, matchNewReceiverCall, newReceiverClass, NEW_RECEIVER_SHAPE, isGuessedRecordAccessor, matchKotlinReceiverChain } from './name-matcher'; import { isVisibleCppMacro, clearCppMacroVisibility } from './cpp-macro-visibility'; import { isCppConstructorRef, matchCppConstructor } from './cpp-constructor'; import { isCppSupertypeRef, matchCppSupertype, clearCppSupertypeMemos } from './cpp-supertypes'; @@ -42,7 +42,7 @@ import { matchShopifyThemeFile } from './shopify-themes'; import { isJsSuperCallLanguage, isJsSuperCallSite, matchJsSuperCall } from './js-super-calls'; import { CLASS_EXPRESSION_REF_LANGUAGES, isCommonJsExportedClass, isJsClassExpression, writtenIn, type ClassExpressionIndex } from './js-class-expressions'; import { matchRubyConstantPathRef } from './ruby-constant-paths'; -import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClassRef, resolveTsQualifiedTypeRef, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isDartImportRef, isLuaRequireRef, isJsPathImportRef, isBoundToOutOfRepoImport, reachesAmbientModule, importBindingTail, clearImportResolverMemos, resolveImportPath, isExternalImport, ESM_IMPORT_LANGUAGES, jsModuleFile, pythonRootFingerprint, pythonReopenScope, pythonPackageImporters, moduleDefaultExport, isJsFamilyFile, isModuleBindingName } from './import-resolver'; +import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClassRef, resolveTsQualifiedTypeRef, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isDartImportRef, isLuaRequireRef, isJsPathImportRef, isBoundToOutOfRepoImport, reachesAmbientModule, importBindingTail, clearImportResolverMemos, resolveImportPath, isExternalImport, pythonImportsInScope, ESM_IMPORT_LANGUAGES, jsModuleFile, pythonRootFingerprint, pythonReopenScope, pythonPackageImporters, moduleDefaultExport, isJsFamilyFile, isModuleBindingName } from './import-resolver'; import { ResolverPool, minRefsForPool, shouldEngageAdaptively } from './resolver-pool'; import { resolveAliasBinding } from './alias-binding'; import { detectFrameworks, getResolvingFrameworks } from './frameworks'; @@ -3327,9 +3327,20 @@ export class ReferenceResolver { * Resolving the specifier is the same question resolveViaImport will ask * next, so a receiver that passes here is one the qualified path can actually * serve; anything else stays a silent miss rather than a wrong edge. + * + * Asked of the binding the reference reads, as resolveViaImport asks it, for + * a receiver the file imports at all: an `import a.b as receiver` inside the + * calling function (#2600), or the file's imports of the name unless the + * function binds it itself — a parameter or local named like the module + * (`def save(store): store.write()`) is a value, and its builtin-named call + * stays filtered. */ private isPythonProjectModule(ref: UnresolvedRef, receiver: string): boolean { - for (const imp of this.context.getImportMappings(ref.filePath, ref.language)) { + const imports = this.context.getImportMappings(ref.filePath, ref.language); + if (!imports.some((imp) => imp.localName === receiver)) return false; + const local = pythonFunctionModuleAlias(receiver, ref, this.context); + if (local) return resolveImportPath(local, ref.filePath, ref.language!, this.context) !== null; + for (const imp of pythonImportsInScope(ref, imports, this.context)) { if (imp.localName !== receiver) continue; // `import pkg.mod` / `import pkg.mod as m` binds the module `source` // names. `from pkg import mod` binds `pkg.mod`, and `from . import mod` diff --git a/src/resolution/name-matcher.ts b/src/resolution/name-matcher.ts index afa8514e5..d1fe6f7df 100644 --- a/src/resolution/name-matcher.ts +++ b/src/resolution/name-matcher.ts @@ -649,6 +649,332 @@ export function pythonFunctionModuleAlias(name: string, ref: UnresolvedRef, cont return null; } +/** + * How the code around a Python reference binds `name`, the receiver of a + * dotted reference whose file imports that name: + * - `'module'`: nothing nearer does (or the function declares it `global`), + * so the file's own import is what the reference reads; + * - `'local'`: the calling function, a function enclosing it, or the + * reference's own statement binds it otherwise — a parameter, an + * assignment (annotated and augmented ones too), a `for` / `with … as` / + * `except … as` target, a walrus, `del`, a nested `def` or `class`, a + * `case` capture, an `import` statement, or a lambda parameter or a + * comprehension variable the reference sits inside; + * - `{ source, exportedName }`: a function-local `from source import + * exportedName as name`, the one import that binds it there; + * - `{ module }`: a function-local `import module as name`, or `import name` + * / `import name.sub` (which bind the top-level `name`). + * A keyword argument, an attribute or subscript store, a comparison, text in + * a string or a comment, and a nested function's own body bind nothing. + * `def by_param(other): other.update(…)` and `with open(p) as other:` read + * no module the file imports as `other`. + */ +export type PythonReceiverBinding = + | 'module' + | 'local' + | { source: string; exportedName: string } + | { module: string }; + +export function pythonReceiverBinding(name: string, ref: UnresolvedRef, context: ResolutionContext): PythonReceiverBinding { + if (ref.language !== 'python' || !/^[A-Za-z_]\w*$/.test(name)) return 'module'; + if (pythonStatementBindsAt(name, ref, context)) return 'local'; + // Innermost first: a closure reads the binding of the function nearest it. + const { functions, innermost, parent } = pythonFunctionIndex(ref.filePath, context); + for (let i = innermost[ref.line] ?? -1; i >= 0; i = parent[i]!) { + const scope = functions[i]!; + const binding = pythonNameScan(context, `binding\0${scope.id}\0${name}`, () => pythonScopeBinding(scope, name, context)); + if (binding !== null) return binding; + } + return 'module'; +} + +/** + * A file's functions and methods, the innermost one around each line (-1 for + * none) and the one around each function's own `def` line, built once per file. + */ +function pythonFunctionIndex(filePath: string, context: ResolutionContext): { functions: Node[]; innermost: Int32Array; parent: Int32Array } { + return pythonNameScan(context, `functions\0${filePath}`, () => { + // Outer before inner: by start, the longer first. + const functions = context.getNodesInFile(filePath) + .filter((n) => n.kind === 'function' || n.kind === 'method') + .sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine); + const innermost = new Int32Array(functions.reduce((m, f) => Math.max(m, f.endLine), 0) + 2).fill(-1); + const parent = new Int32Array(functions.length).fill(-1); + functions.forEach((f, i) => { + parent[i] = innermost[f.startLine]!; + innermost.fill(i, f.startLine, f.endLine + 1); + }); + return { functions, innermost, parent }; + }); +} + +/** How the function `scope` itself binds `name`; null when it doesn't, and an enclosing scope decides. */ +function pythonScopeBinding(scope: Node, name: string, context: ResolutionContext): PythonReceiverBinding | null { + const { params, statements, nested } = pythonScopeText(scope, context); + if (statements.some((st) => pythonDeclaresGlobal(st, name))) return 'module'; + if (params.includes(name) || nested.includes(name)) return 'local'; + let imported: Exclude | null = null; + for (const st of statements) { + const binding = pythonImportBinds(st, name); + if (binding === undefined) { + if (pythonStatementBinds(st, name)) return 'local'; + } else if (binding) { + // Two imports binding the name differently: which one a reference reads + // depends on the path the function took. + if (imported && JSON.stringify(imported) !== JSON.stringify(binding)) return 'local'; + imported = binding; + } + } + return imported; +} + +/** + * The parameter names of the function `scope`, the statements of its own + * body (continuation lines joined, strings blanked, split on `;`), and the + * names its nested `def`s and `class`es bind, whose bodies are not its own. + */ +function pythonScopeText(scope: Node, context: ResolutionContext): { params: string[]; statements: string[]; nested: string[] } { + return pythonNameScan(context, `scope\0${scope.id}`, () => { + const lines = pythonMemberLines(scope.filePath, context); + const last = Math.min(scope.endLine, lines.length); + const indent = (l: string) => l.length - l.trimStart().length; + const statements: string[] = []; + const nested: string[] = []; + let header: string | null = null; + let current: string | null = null; + let depth = 0; + let continued = false; + let nestedIndent = -1; + const flush = () => { + if (current === null) return; + if (header === null) header = current; + else statements.push(...current.split(';').map((s) => s.trim()).filter(Boolean)); + current = null; + }; + for (let k = scope.startLine; k <= last; k++) { + const raw = lines[k - 1] ?? ''; + const blanked = pythonBlankStrings(raw); + const inside = depth > 0 || continued; + for (const c of blanked) { + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') depth = Math.max(0, depth - 1); + } + continued = /\\\s*$/.test(blanked); + if (inside) { + if (current !== null) current += ' ' + blanked.trim().replace(/\\$/, ''); + continue; + } + if (!blanked.trim()) continue; + flush(); + if (nestedIndent >= 0) { + if (indent(raw) > nestedIndent) continue; + nestedIndent = -1; + } + const text = blanked.trim().replace(/\\$/, ''); + // The scope's first line is its own `def` header. + const def = k === scope.startLine ? null : /^(?:async\s+)?(?:def|class)\s+(\w+)/.exec(text); + if (def) { + nested.push(def[1]!); + nestedIndent = indent(raw); + continue; + } + current = text; + } + flush(); + // `def name[T](a, *b, c: int = 1, **d)`: the names between the brackets. + const head = header ?? ''; + const open = /\bdef\s+\w+\s*(?:\[[^\]]*\]\s*)?\(/.exec(head); + const params = open ? pythonParamNames(head.slice(open.index + open[0].length, pythonClosingParen(head, open.index + open[0].length - 1))) : []; + return { params, statements, nested }; + }); +} + +/** The index of the bracket closing the one at `open`, or the text's end. */ +function pythonClosingParen(text: string, open: number): number { + let depth = 0; + for (let i = open; i < text.length; i++) { + const c = text[i]!; + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') { + depth--; + if (depth === 0) return i; + } + } + return text.length; +} + +/** `text` split at its top-level commas (none inside brackets). */ +function pythonTopLevelParts(text: string): string[] { + const parts: string[] = []; + let depth = 0; + let from = 0; + for (let i = 0; i < text.length; i++) { + const c = text[i]!; + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') depth = Math.max(0, depth - 1); + else if (c === ',' && depth === 0) { + parts.push(text.slice(from, i)); + from = i + 1; + } + } + parts.push(text.slice(from)); + return parts; +} + +/** The names a parameter list binds: `a`, `*b`, `c: int = 1`, `**d`; never the bare `*` or `/` markers. */ +function pythonParamNames(list: string): string[] { + return pythonTopLevelParts(list) + .map((p) => /^\s*\*{0,2}\s*([A-Za-z_]\w*)/.exec(p)?.[1]) + .filter((p): p is string => p !== undefined); +} + +/** `global name` (or a list naming it). */ +function pythonDeclaresGlobal(statement: string, name: string): boolean { + const m = /^global\s+(.+)$/.exec(statement); + return !!m && m[1]!.split(',').some((n) => n.trim() === name); +} + +/** + * What an import statement binds `name` to: the module of `import a.b as + * name`, or the top-level `name` of `import name` / `import name.sub`; the + * source and name of `from a import b as name`; null for an import that + * doesn't bind it, undefined for any other statement. + */ +function pythonImportBinds(statement: string, name: string): Exclude | null | undefined { + const plain = /^import\s+([\s\S]+)$/.exec(statement); + if (plain) { + for (const part of plain[1]!.replace(/[()]/g, ' ').split(',')) { + const m = /^([\w.]+)(?:\s+as\s+(\w+))?$/.exec(part.trim()); + if (!m) continue; + if (m[2] === name) return { module: m[1]! }; + if (!m[2] && m[1]!.split('.')[0] === name) return { module: name }; + } + return null; + } + const from = /^from\s+(\.*[\w.]*)\s+import\s+([\s\S]+)$/.exec(statement); + if (!from) return undefined; + for (const part of from[2]!.replace(/[()]/g, ' ').split(',')) { + const m = /^(\w+)(?:\s+as\s+(\w+))?$/.exec(part.trim()); + if (m && (m[2] ?? m[1]) === name) return { source: from[1]!, exportedName: m[1]! }; + } + return null; +} + +/** Whether one statement (strings blanked) binds `name` other than by an import. */ +function pythonStatementBinds(statement: string, name: string): boolean { + const n = escapeRegExp(name); + if (pythonAssignTargets(statement).some((t) => pythonTargetBinds(t, name))) return true; + // `name: T` declares a local without assigning it. + if (new RegExp(`^${n}\\s*:(?!=)`).test(statement)) return true; + const loop = /^(?:async\s+)?for\s+([\s\S]+?)\s+in\b/.exec(statement); + if (loop && pythonTargetBinds(loop[1]!, name)) return true; + // `with … as name`, `except … as name`, `case … as name`, `as (a, name)`. + for (const m of statement.matchAll(/\bas\s+(\([^)]*\)|[\w.]+)/g)) if (pythonTargetBinds(m[1]!, name)) return true; + if (new RegExp(`(? pythonTargetBinds(t, name))) return true; + return pythonCaseCaptures(statement, name); +} + +/** + * Whether a `case` clause's pattern captures `name`: written bare, not as a + * value pattern's dotted part (`other.MODE`), a class pattern's class + * (`other(…)`) or a keyword pattern's attribute (`Point(other=1)`). `case = 1` + * assigns a variable named `case`. + */ +function pythonCaseCaptures(statement: string, name: string): boolean { + const m = /^case\b([\s\S]*)$/.exec(statement); + if (!m || pythonAssignTargets(statement).length > 0) return false; + let depth = 0; + let end = -1; + for (let i = 0; i < m[1]!.length && end < 0; i++) { + const c = m[1]![i]!; + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') depth = Math.max(0, depth - 1); + else if (c === ':' && depth === 0) end = i; + } + if (end < 0) return false; + const pattern = m[1]!.slice(0, end).replace(/\bif\b[\s\S]*$/, ''); + for (const hit of pattern.matchAll(new RegExp(`(?= lines.length) return false; + let first = at; + while (first > 0 && !starts[first]) first--; + let end = at; + while (end + 1 < lines.length && !starts[end + 1]) end++; + let text = ''; + let offset = 0; + for (let i = first; i <= end; i++) { + if (i === at) offset = text.length + Math.max(0, ref.column ?? 0); + text += pythonBlankStrings(lines[i]!) + '\n'; + } + if (!/\b(?:lambda|for)\b/.test(text)) return false; + // A lambda's body runs from its `:` to the first comma or closing bracket of + // its own level. + for (const m of text.matchAll(/\blambda\b/g)) { + const from = m.index! + 6; + let depth = 0; + let colon = -1; + for (let i = from; i < text.length && colon < 0; i++) { + const c = text[i]!; + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') { if (depth === 0) break; depth--; } + else if (c === ':' && depth === 0) colon = i; + } + if (colon < 0 || offset <= colon || !pythonParamNames(text.slice(from, colon)).includes(name)) continue; + let stop = text.length; + depth = 0; + for (let i = colon + 1; i < text.length; i++) { + const c = text[i]!; + if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') { if (depth === 0) { stop = i; break; } depth--; } + else if (c === ',' && depth === 0) { stop = i; break; } + } + if (offset < stop) return true; + } + // A comprehension's `for` sits inside the brackets it binds in. + const opens: number[] = []; + const pairs: Array<[number, number]> = []; + for (let i = 0; i < text.length; i++) { + const c = text[i]!; + if (c === '(' || c === '[' || c === '{') opens.push(i); + else if ((c === ')' || c === ']' || c === '}') && opens.length) pairs.push([opens.pop()!, i]); + } + for (const o of opens) pairs.push([o, text.length]); + for (const m of text.matchAll(/\bfor\b/g)) { + const around = pairs.filter(([o, c]) => o < m.index! && c > m.index!).sort((a, b) => b[0] - a[0])[0]; + if (!around || offset <= around[0] || offset >= around[1]) continue; + const targets = /^\s*([\s\S]+?)\s+in\b/.exec(text.slice(m.index! + 3, around[1])); + if (targets && pythonTargetBinds(targets[1]!, name)) return true; + } + return false; +} + +/** + * The imports binding `name` at module scope, or in a function declaring it + * `global`, keyed as pythonLineBindings keys them: `from a import x as name` + * → `a:x`, `import a.b as name` → `a.b:*`. + */ +export function pythonModuleImportKeys(name: string, filePath: string, context: ResolutionContext): Set { + return new Set(pythonGlobalBindings(name, filePath, context).flatMap((b) => (b.kind === 'import' ? [b.key] : []))); +} + /** Distinct sources a file imports `name` from, at any scope (`from a import x` → `a:x`). */ function pythonImportKeys(name: string, filePath: string, context: ResolutionContext): Set { return pythonNameScan(context, `imports\0${filePath}\0${name}`, () => scanPythonImportKeys(name, filePath, context));