Repository navigation
Expand file tree
/
Copy pathcodegraph.ts
More file actions
3104 lines (2861 loc) · 135 KB
/
Copy pathcodegraph.ts
File metadata and controls
3104 lines (2861 loc) · 135 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* CodeGraph
*
* A local-first code intelligence system that builds a semantic
* knowledge graph from any codebase.
*/
import * as fs from 'fs';
import * as path from 'path';
import { readFile } from 'fs/promises';
import { extractReactRouterConfig } from './resolution/frameworks/react-router';
import {
Node,
NodeKind,
Edge,
EdgeKind,
FileRecord,
ExtractionResult,
Subgraph,
TraversalOptions,
SearchOptions,
SearchResult,
SegmentMatch,
Context,
GraphStats,
TaskInput,
TaskContext,
BuildContextOptions,
FindRelevantContextOptions,
UnresolvedReference,
IndexHealth,
Language,
} from './types';
import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
import { WalCheckpointValve, resolveWalValveMb } from './db/wal-valve';
import { QueryBuilder, type NodeSpan } from './db/queries';
import { importPathKeys, moduleReferenceKeys } from './db/reference-tail';
import {
isInitialized,
createDirectory,
removeDirectory,
validateDirectory,
} from './directory';
import type {
ExtractionOrchestrator,
IndexProgress,
IndexResult,
SyncResult,
} from './extraction';
import type { ReferenceResolver, ResolutionResult } from './resolution';
import { GraphTraverser, GraphQueryManager } from './graph';
import { findNamedCopybooks, type NamedCopybook } from './graph/cobol-copybooks';
import { ContextBuilder, createContextBuilder } from './context';
import { Mutex, FileLock } from './utils';
import { logWarn } from './errors';
import type { FileWatcher, WatchOptions, PendingFile } from './sync';
import { EXTRACTION_VERSION } from './extraction/extraction-version';
import { isGeneratedFile } from './extraction/generated-detection';
import { hasGrammarLoadFailure, isFileLevelOnlyLanguage } from './extraction/grammars';
import { refreshNearDuplicates } from './graph/near-duplicates';
import { getCodeGraphDir } from './directory';
import { deriveProjectNameTokens } from './search/query-utils';
import { findSeedSource, copySeedIndex, type SeedSource } from './sync/worktree-seed';
import ignore from 'ignore';
import { loadDeprioritizePatterns } from './project-config';
import { CodeGraphPackageVersion } from './mcp/version';
import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments';
import { seedLiteralsInQuery, seedRouteNamesInQuery } from './extraction/literal-capture';
import { createYielder } from './resolution/cooperative-yield';
import { createImportReader, reopenNameMatchedImports, reopenReferencesThroughAddedFiles, reopenReferencesThroughBarrels, reopenReferencesThroughChangedDefaults, reopenReferencesThroughDeclaredPackages } from './resolution/module-importers';
import { minRefsForPool } from './resolution/resolver-pool';
/** The extraction, resolution and watcher modules, loaded on first use. */
interface Stacks {
extraction: typeof import('./extraction');
resolution: typeof import('./resolution');
sync: typeof import('./sync');
}
let stacks: Stacks | undefined;
/**
* The public entry (`./index`) hands over the stacks it already imports. A
* read-only caller that loads this module directly — a query worker, the MCP
* engine — never parses, resolves or watches until it has to, and skipping
* those modules saves each worker thread tens of MB.
*/
export function provideStacks(s: Stacks): void {
stacks = s;
}
const loadStacks = (): Stacks =>
(stacks ??= { extraction: require('./extraction'), resolution: require('./resolution'), sync: require('./sync') });
const extraction = () => loadStacks().extraction;
const resolution = () => loadStacks().resolution;
const sync = () => loadStacks().sync;
/**
* Options for initializing a new CodeGraph project
*/
export interface InitOptions {
/** Whether to run initial indexing after init */
index?: boolean;
/** Progress callback for indexing */
onProgress?: (progress: IndexProgress) => void;
}
/**
* Options for opening an existing CodeGraph project
*/
export interface OpenOptions {
/** Whether to run sync if files have changed */
sync?: boolean;
/** Whether to run in read-only mode */
readOnly?: boolean;
}
/**
* Options for indexing
*/
export interface IndexOptions {
/** Progress callback */
onProgress?: (progress: IndexProgress) => void;
/** Abort signal for cancellation */
signal?: AbortSignal;
/** Enable verbose logging (worker lifecycle, memory, timeouts) */
verbose?: boolean;
/** Watcher fast path: reconcile ONLY these project-relative paths (see ExtractionOrchestrator.sync). */
paths?: string[];
/**
* Refresh synthesized edges after the sync returns, debounced, instead of
* inside it (see scheduleSynthesisRefresh). The watcher sets this so a save
* is searchable as fast as before; one-shot syncs refresh inline.
*/
deferSynthesis?: boolean;
/** Sync must finish queued synthesis and reject failed file extraction or incomplete synthesis. */
requireComplete?: boolean;
}
/**
* Main CodeGraph class
*
* Provides the primary interface for interacting with the code knowledge graph.
*/
/** project_metadata key for the Python package-root digest (see sync). */
const PYTHON_ROOTS_METADATA_KEY = 'python_root_fingerprint';
export class CodeGraph {
private db: DatabaseConnection;
private queries: QueryBuilder;
private projectRoot: string;
// Assigned via wireLayers() from the constructor (and again on reopen) — the
// `!` tells TS these are definitely set even though the assignment is one
// method call away from the constructor body.
private lazyOrchestrator?: ExtractionOrchestrator;
private lazyResolver?: ReferenceResolver;
private get orchestrator(): ExtractionOrchestrator {
return (this.lazyOrchestrator ??= new (extraction().ExtractionOrchestrator)(this.projectRoot, this.queries));
}
private get resolver(): ReferenceResolver {
return (this.lazyResolver ??= resolution().createResolver(this.projectRoot, this.queries));
}
/**
* Detect frameworks against the files as they are now. A resolver created
* by this call has just detected them, so detecting again would repeat the
* same work (a full read of every source file for some detectors).
*/
private redetectFrameworks(): ReferenceResolver {
if (!this.lazyResolver) return this.resolver;
this.lazyResolver.initialize();
return this.lazyResolver;
}
private graphManager!: GraphQueryManager;
private traverser!: GraphTraverser;
private contextBuilder!: ContextBuilder;
// Mutex for preventing concurrent indexing operations (in-process)
private indexMutex = new Mutex();
// File lock for preventing concurrent writes across processes (CLI, MCP, git hooks)
private fileLock: FileLock;
/** Files whose synthesized edges await a refresh (deferred incremental syncs). */
private synthesisDirty = new Set<string>();
private synthesisTimer: ReturnType<typeof setTimeout> | null = null;
/** How long the last refresh took; the debounce grows with it (see scheduleSynthesisRefresh). */
private lastSynthesisMs = 0;
// File watcher for auto-sync on file changes
private watcher: FileWatcher | null = null;
private constructor(
db: DatabaseConnection,
queries: QueryBuilder,
projectRoot: string
) {
this.db = db;
this.queries = queries;
this.projectRoot = projectRoot;
this.fileLock = new FileLock(
path.join(getCodeGraphDir(projectRoot), 'codegraph.lock')
);
this.wireLayers();
}
/**
* (Re)build the query/extraction/graph layers over the current `this.queries`
* (which wraps `this.db`). Factored out of the constructor so `reopenIfReplaced`
* can rebuild them against a fresh connection without duplicating the wiring.
* The path-based `fileLock` is independent of the DB handle, so it stays put.
*/
private wireLayers(): void {
// Down-weight the project name as a query term in search ranking — it names
// the whole repo, not a symbol, so it has no discriminative value (#720).
try {
this.queries.setProjectNameTokens(deriveProjectNameTokens(this.projectRoot));
} catch {
// Best-effort: ranking still works without it.
}
// Down-weight the peripheral trees the project named in `codegraph.json`
// `deprioritize` — indexed and findable, but never outranking real code
// (#982). Ranking-only, so a bad pattern costs relevance, never recall.
//
// Read LAZILY, not once here: `wireLayers` runs from the constructor and
// from `reopenIfReplaced`, so a matcher built here would freeze at whatever
// the config said when the project opened. The MCP server caches one
// CodeGraph per root for its whole lifetime, so editing `codegraph.json`
// would appear to do nothing until the process restarted — `exclude` and
// `include` do not behave that way. The provider runs once per ranking
// pass (callers reuse the predicate for every path in it), and
// `loadDeprioritizePatterns` is mtime-cached, so a pass costs one `stat`;
// the compiled matcher is memoized on the pattern array's identity, which
// the cache keeps stable. No patterns yields no predicate at all.
let cachedPatterns: string[] | undefined;
let cachedMatcher: ReturnType<typeof ignore> | undefined;
this.queries.setDeprioritizedPathProvider(() => {
try {
const patterns = loadDeprioritizePatterns(this.projectRoot);
if (patterns.length === 0) return undefined;
if (patterns !== cachedPatterns) {
cachedPatterns = patterns;
cachedMatcher = ignore().add(patterns);
}
const matcher = cachedMatcher!;
return (filePath: string): boolean => {
try {
const rel = path.isAbsolute(filePath)
? path.relative(this.projectRoot, filePath)
: filePath;
if (!rel || rel.startsWith('..')) return false;
return matcher.ignores(rel.split(path.sep).join('/'));
} catch {
// Ranking must never take the search down with it.
return false;
}
};
} catch {
return undefined;
}
});
// Built on first use against the current `queries` (see the getters).
this.lazyOrchestrator = undefined;
this.lazyResolver = undefined;
this.graphManager = new GraphQueryManager(this.queries);
this.traverser = new GraphTraverser(this.queries);
this.contextBuilder = createContextBuilder(
this.projectRoot,
this.queries,
this.traverser
);
}
/**
* Set when this instance followed a database replaced on disk (#1902): the
* next sync that can run reconciles the whole tree, because whatever the old
* handle absorbed since the rebuild never reached the new file.
*/
private pendingFullReconcile = false;
/** How long a recreated, not-yet-indexed database is treated as a rebuild in progress. */
private static readonly RECREATE_GRACE_MS = 120_000;
/** The database file at the path was written within the recreate grace window. */
private isFreshlyRecreated(): boolean {
try {
return Date.now() - fs.statSync(getDatabasePath(this.projectRoot)).mtimeMs < CodeGraph.RECREATE_GRACE_MS;
} catch {
return false;
}
}
/**
* Heal a stale database handle in place. If `.codegraph/` was removed and
* recreated at the SAME path while this instance held the DB open — a git
* worktree removed and re-added, or `rm -rf .codegraph` + `codegraph init` —
* our open fd points at the now-unlinked inode and can never see the new
* index, so every query returns the pre-removal snapshot until the process
* restarts (#925). When that's detected, open the live file at the same path,
* rebuild the query layers, and swap them IN PLACE, so every holder of this
* instance (the MCP daemon's default project, cached projectPath connections)
* heals without a restart. Returns true iff it reopened.
*
* POSIX-only in practice: `isReplacedOnDisk` never fires on Windows (an open
* file can't be unlinked there, and st_ino is unreliable).
*
* Refuses (returns false) while an index/sync holds the index mutex: closing
* the handle that run is writing through would break it mid-flight. `sync()`
* performs the same check itself once it holds the mutex (#1902), so the
* replaced file is still picked up — by that sync, or by the caller's retry.
*/
reopenIfReplaced(): boolean {
if (this.indexMutex.isLocked()) return false;
return this.reopenReplacedDatabase();
}
/** The body of {@link reopenIfReplaced}, without the in-flight-sync guard. */
private reopenReplacedDatabase(): boolean {
if (!this.db.isReplacedOnDisk()) return false;
const dbPath = this.db.getPath();
// Open the live file FIRST — if that throws (e.g. mid-recreate), the old
// handle stays in place and the caller retries on the next query, rather
// than leaving this instance with no connection at all.
const fresh = DatabaseConnection.open(dbPath, { readOnly: this.db.readOnly });
const stale = this.db;
this.db = fresh;
this.queries = new QueryBuilder(fresh.getDb());
this.wireLayers();
// Whoever reopened — a sync, or a tool call's self-heal — the next sync
// must reconcile the whole tree (#1902).
this.pendingFullReconcile = true;
// Releasing the dead handle also frees the leaked db/-wal/-shm fds that were
// pinning the unlinked inode (#925).
try { stale.close(); } catch { /* the old inode is gone; closing just frees fds */ }
return true;
}
// ===========================================================================
// Lifecycle Methods
// ===========================================================================
/**
* Initialize a new CodeGraph project
*
* Creates the .CodeGraph directory, database, and configuration.
*
* @param projectRoot - Path to the project root directory
* @param options - Initialization options
* @returns A new CodeGraph instance
*/
static async init(projectRoot: string, options: InitOptions = {}): Promise<CodeGraph> {
await extraction().initGrammars();
const resolvedRoot = path.resolve(projectRoot);
// Check if already initialized
if (isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
}
// Create directory structure
createDirectory(resolvedRoot);
// Initialize database
const dbPath = getDatabasePath(resolvedRoot);
const db = DatabaseConnection.initialize(dbPath);
const queries = new QueryBuilder(db.getDb());
const instance = new CodeGraph(db, queries, resolvedRoot);
// Run initial indexing if requested
if (options.index) {
await instance.indexAll({ onProgress: options.onProgress });
}
return instance;
}
/**
* Initialize synchronously (without indexing)
*/
static initSync(projectRoot: string): CodeGraph {
const resolvedRoot = path.resolve(projectRoot);
// Check if already initialized
if (isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
}
// Create directory structure
createDirectory(resolvedRoot);
// Initialize database
const dbPath = getDatabasePath(resolvedRoot);
const db = DatabaseConnection.initialize(dbPath);
const queries = new QueryBuilder(db.getDb());
return new CodeGraph(db, queries, resolvedRoot);
}
/**
* Initialize a git worktree from a sibling worktree's index: copy the
* closest compatible one (see `sync/worktree-seed.ts`), then sync it to this
* tree, which re-parses only the files that differ. Returns null, leaving
* nothing behind, when no sibling qualifies or the copy or sync fails; the
* caller then indexes from scratch.
*/
static async initFromSibling(
projectRoot: string,
options: Pick<IndexOptions, 'onProgress'> = {}
): Promise<{ codegraph: CodeGraph; source: SeedSource; sync: SyncResult } | null> {
const resolvedRoot = path.resolve(projectRoot);
if (isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
}
const source = findSeedSource(resolvedRoot);
if (!source) return null;
createDirectory(resolvedRoot);
let instance: CodeGraph | null = null;
try {
copySeedIndex(source, resolvedRoot);
instance = await CodeGraph.open(resolvedRoot);
const sync = await instance.sync({ onProgress: options.onProgress });
return { codegraph: instance, source, sync };
} catch {
instance?.close();
removeDatabaseFiles(getDatabasePath(resolvedRoot));
return null;
}
}
/**
* Open an existing CodeGraph project
*
* @param projectRoot - Path to the project root directory
* @param options - Open options
* @returns A CodeGraph instance
*/
static async open(projectRoot: string, options: OpenOptions = {}): Promise<CodeGraph> {
const resolvedRoot = path.resolve(projectRoot);
// Check if initialized
if (!isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
}
// Validate directory structure
const validation = validateDirectory(resolvedRoot);
if (!validation.valid) {
throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
}
// Open database. The secondary indexes an interrupted bulk load dropped are
// rebuilt off this thread, and finished before the instance is returned
// (#1887): rebuilt in line on a large index, they blocked the thread longer
// than the liveness watchdog allows, and every restart was killed the same
// way.
const dbPath = getDatabasePath(resolvedRoot);
const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly, deferIndexRecovery: true });
try {
await db.recoverIndexesOffThread();
} catch (err) {
db.close();
throw err;
}
const queries = new QueryBuilder(db.getDb());
const instance = new CodeGraph(db, queries, resolvedRoot);
// Sync if requested (a read-only connection cannot write)
if (options.sync && !options.readOnly) {
try {
await instance.sync();
} catch (err) {
instance.destroy();
throw err;
}
}
return instance;
}
/**
* Rebuild the project's database from scratch and return a fresh, empty
* instance — the "same result as a fresh init" semantics that `codegraph
* index` documents.
*
* Unlike `open()` followed by `clear()`, this DISCARDS the existing
* `.codegraph/codegraph.db` (and its `-wal`/`-shm` sidecars) before
* re-initializing, instead of opening the old database and DELETE-ing every
* row. On a large or pre-fix poisoned index — e.g. an old graph that scanned
* an ignored gitlink corpus (#1065) into ~1.6M nodes with a multi-GB WAL —
* the per-row `nodes_fts` delete-trigger churn blocks the main thread long
* enough to trip the #850 liveness watchdog before indexing even starts, so a
* full re-index could never recover the bad state (#1067). Discarding the
* files is O(1) regardless of size, reclaims the disk, and sidesteps opening
* (and running migrations against) the poisoned database entirely.
*/
static async recreate(projectRoot: string): Promise<CodeGraph> {
await extraction().initGrammars();
const resolvedRoot = path.resolve(projectRoot);
// Check if initialized — recreate REBUILDS an existing project; it is not a
// first-time `init`.
if (!isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
}
const dbPath = getDatabasePath(resolvedRoot);
try {
removeDatabaseFiles(dbPath);
} catch (err) {
// POSIX unlinks an open file fine; this fires mainly on Windows when a
// live daemon/MCP server still holds the database. Turn the raw EBUSY into
// an actionable instruction instead of a generic failure.
const reason = err instanceof Error ? err.message : String(err);
throw new Error(
`Could not rebuild the index — the database file is in use (${reason}). ` +
`Stop any running CodeGraph MCP server/daemon for this project and retry, ` +
`or remove the ${getCodeGraphDir(resolvedRoot)} directory and run "codegraph init".`
);
}
// Re-create an empty, freshly-schema'd database at the same path.
const db = DatabaseConnection.initialize(dbPath);
const queries = new QueryBuilder(db.getDb());
return new CodeGraph(db, queries, resolvedRoot);
}
/**
* Open synchronously (without sync).
*
* `deferIndexRecovery` leaves the secondary indexes an interrupted bulk load
* dropped for {@link recoverIndexes} to rebuild off this thread, instead of
* rebuilding them here (#1887): for a caller on a watched event loop, which
* then awaits {@link recoverIndexes} before it relies on the project.
*/
static openSync(projectRoot: string, options: Pick<OpenOptions, 'readOnly'> & { deferIndexRecovery?: boolean; compactConnection?: boolean } = {}): CodeGraph {
const resolvedRoot = path.resolve(projectRoot);
// Check if initialized
if (!isInitialized(resolvedRoot)) {
throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
}
// Validate directory structure
const validation = validateDirectory(resolvedRoot);
if (!validation.valid) {
throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
}
// Open database
const dbPath = getDatabasePath(resolvedRoot);
const db = DatabaseConnection.open(dbPath, { readOnly: options.readOnly, deferIndexRecovery: options.deferIndexRecovery, compactConnection: options.compactConnection });
const queries = new QueryBuilder(db.getDb());
return new CodeGraph(db, queries, resolvedRoot);
}
/**
* Rebuild, on a worker thread, the secondary indexes an interrupted bulk
* load dropped and `openSync({ deferIndexRecovery: true })` left (#1887).
* Resolves at once when nothing is left, which is every open but the first
* after such an interruption; rejects when the rebuild fails, and the next
* call tries again. Reads stay correct meanwhile, only slower.
*/
recoverIndexes(): Promise<void> {
return this.db.recoverIndexesOffThread();
}
/** Whether {@link recoverIndexes} still has indexes to rebuild. */
hasPendingIndexRecovery(): boolean {
return this.db.hasDeferredIndexRecovery();
}
/** The query tokens the index has never seen (no node, edge or file carries them). */
getUnseenNames(names: readonly string[]): string[] {
return this.queries.getUnseenNames(names);
}
/**
* Check if a directory has been initialized as a CodeGraph project
*/
static isInitialized(projectRoot: string): boolean {
return isInitialized(path.resolve(projectRoot));
}
/**
* Close the CodeGraph instance and release resources
*/
close(): void {
if (this.synthesisTimer) { clearTimeout(this.synthesisTimer); this.synthesisTimer = null; }
this.unwatch();
// Release file lock if held
this.fileLock.release();
this.db.close();
}
/**
* Get the project root directory
*/
getProjectRoot(): string {
return this.projectRoot;
}
// ===========================================================================
// Indexing
// ===========================================================================
/**
* Index all files in the project
*
* Uses a mutex to prevent concurrent indexing operations.
*/
async indexAll(options: IndexOptions = {}): Promise<IndexResult> {
return this.indexMutex.withLock(async () => {
try {
this.fileLock.acquire();
} catch {
return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
}
const startedAt = Date.now();
// Defer WAL auto-checkpointing for the whole bulk run (#1231): the
// default 1000-page interval re-writes hot pages into the main DB file
// over and over — ~95% of all disk I/O during a bulk index, and a
// 19+min → 45s difference on HDD-class storage. The valve bounds WAL
// growth by backfilling PASSIVEly on a worker thread (never blocking
// the writer or the #850 watchdog heartbeat); runMaintenance below does
// the final fold-up before the interval is restored in the finally.
// Kill switch: CODEGRAPH_NO_WAL_DEFER=1. Non-WAL journal modes (some
// network filesystems) have no WAL to defer — skip.
// Fast-init: on a COMPLETELY fresh DB, trade crash-durability for speed
// during the bulk build (journal in memory, no fsync). Safe because the
// DB is disposable until the index completes — index_state stays
// 'indexing' and a crashed init is re-run from scratch; existing DBs
// (re-index/sync) never take this path. Kill switch:
// CODEGRAPH_NO_FAST_INIT=1 (same pattern as CODEGRAPH_NO_WAL_DEFER).
const freshDb = this.queries.getNodeAndEdgeCount().nodes === 0;
const fastInit = process.env.CODEGRAPH_NO_FAST_INIT !== '1' && freshDb;
if (fastInit) {
try {
this.db.getDb().pragma('journal_mode = MEMORY');
this.db.getDb().pragma('synchronous = OFF');
} catch { /* keep WAL */ }
}
const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
let walValve: WalCheckpointValve | null = null;
let priorAutocheckpoint = 1000;
// Set when the fastInit+pool path below defers autocheckpointing, so the
// finally knows to restore the interval on that path too.
let restoreAutocheckpoint = false;
if (deferWal) {
priorAutocheckpoint = this.db.getWalAutocheckpoint();
this.db.setWalAutocheckpoint(0);
walValve = new WalCheckpointValve(
this.db,
resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
undefined,
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
);
walValve.start();
}
try {
const gitState = this.orchestrator.beginGitIndexState(true);
const before = this.queries.getNodeAndEdgeCount();
// Mark the index as in-flight BEFORE any writes: a run killed
// mid-index (OOM, SIGKILL, the #850 liveness watchdog) leaves this
// marker behind, so `codegraph status` can tell a truncated index
// from a completed one instead of silently serving partial results.
try { this.queries.setMetadata('index_state', 'indexing'); } catch { /* metadata is advisory */ }
// Segment vocabulary starts empty and is repopulated by the node write
// path as every file (re-)indexes below — so a full index is also the
// orphan-cleanup pass for names deleted since the last one.
try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ }
try { this.queries.clearLiterals(); } catch { /* literals are repopulated even for unchanged files */ }
// Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync
// triggers, rebuild nodes_fts once from the nodes table afterwards.
// Crash inside the window is healed on the next DatabaseConnection.open.
this.db.beginBulkNodeLoad();
// Fresh-init only: also drop the parse-lane secondary indexes for the
// mass insert (the store-writer's B-tree-maintenance floor, plan §4d)
// and rebuild each in one scan afterwards. Incremental runs keep them
// — they delete per-file rows mid-phase through the file_path indexes.
if (freshDb) this.db.beginBulkParseLoad();
let result: IndexResult;
try {
result = await this.orchestrator.indexAll(
options.onProgress,
options.signal,
options.verbose,
walValve ? () => walValve!.backpressure() : undefined,
// Store-writer offload is fresh-DB-only: with any pre-existing
// data the store path must read (existing-file checks, cross-file
// edge snapshots) and delete, which belongs on one thread.
freshDb ? { dbPath: this.db.getPath(), fastInit } : null
);
} finally {
if (freshDb) {
const tIdx = Date.now();
await this.db.endBulkParseLoad();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-index-rebuild: ${Date.now() - tIdx}ms`);
}
const tFts = Date.now();
this.db.endBulkNodeLoad();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] fts-rebuild: ${Date.now() - tFts}ms`);
}
// Fold the parse phase's WAL BEFORE the first post-parse reads
// (resolver re-init and resolution both read on the main thread):
// paging a bulk-write-sized WAL there is what blew the #850
// watchdog's 60s window in the #1231 repro. Off-thread + awaited,
// so the event loop keeps turning.
if (walValve) await walValve.foldNow();
// Re-detect frameworks now that the index is populated. The resolver
// is constructed with createResolver() before any files exist, so
// framework resolvers whose detect() consults the indexed file list
// (e.g. UIKit/SwiftUI scanning for imports, swift-objc-bridge looking
// for both Swift and ObjC files) all return false on that initial pass
// and silently drop themselves. Re-initializing here gives them a
// chance to see the actual project before resolution runs.
if (result.success && result.filesIndexed > 0) {
const tReinit = Date.now();
this.redetectFrameworks();
// Cross-file finalization (e.g. NestJS RouterModule prefixes). Runs
// before resolution so updated names show up in subsequent reads.
this.resolver.runPostExtract();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolver-reinit: ${Date.now() - tReinit}ms`);
}
// Resolve references to create call/import/extends edges
if (result.success && result.filesIndexed > 0) {
// Get count without loading all refs into memory
const unresolvedCount = this.queries.getUnresolvedReferencesCount();
// Fast-init leaves the DB in memory-journal (rollback) mode, where
// the parallel resolver pool's read connections would contend with
// the main writer's exclusive commits. When the pool will actually
// run (enough pending refs), restore WAL BEFORE resolution so
// readers never block the writer; otherwise stay in the fast mode
// until the finally — sequential resolution has no readers, and a
// pool that boots mid-run switches to WAL itself (ensureWalForPool).
if (fastInit) {
try {
if (unresolvedCount >= minRefsForPool()) {
this.db.getDb().pragma('synchronous = NORMAL');
this.db.getDb().pragma('journal_mode = WAL');
}
// Defer auto-checkpointing for the resolution phase, same
// rationale as the deferWal path above: at the default 1000-page
// interval, the persist loop's edge inserts + ref deletes make
// SQLite re-write hot B-tree pages into the main DB file inline
// on the writer over and over (#1231's pathology — measured as
// ~58% of the resolution phase on a 255k-ref repo). Armed even
// while still in memory-journal mode, so a mid-run switch to WAL
// gets the same deferral (the interval and the valve are inert
// until a WAL exists). The valve bounds WAL growth off-thread;
// runMaintenance does the final fold and the finally restores
// the interval.
priorAutocheckpoint = this.db.getWalAutocheckpoint();
this.db.setWalAutocheckpoint(0);
restoreAutocheckpoint = true;
walValve = new WalCheckpointValve(
this.db,
undefined,
undefined,
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
);
walValve.start();
} catch { /* keep current mode; resolution still works sequentially */ }
}
options.onProgress?.({
phase: 'resolving',
current: 0,
total: unresolvedCount,
});
const tResolve = Date.now();
await this.resolveReferencesBatched(
(current, total) => {
options.onProgress?.({
phase: 'resolving',
current,
total,
});
},
(done, totalPasses) => {
options.onProgress?.({
phase: 'linking',
current: done,
total: totalPasses,
});
},
walValve
);
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolution: ${Date.now() - tResolve}ms`);
// The deferred passes below can lazily re-open a kernel conn —
// rusqlite's wal-index locks can't see node:sqlite's, so the
// off-thread checkpointer must be quiesced while any kernel conn
// could act. Idempotent with the maintenance stop below.
if (walValve) { walValve.stop(); await walValve.drain(); }
// Second pass: chained calls whose method lives on a supertype the
// receiver conforms to (protocol-extension / inherited / default-
// interface). Needs the implements/extends edges the main pass just
// built, so it runs after resolution (#750).
const tChained = Date.now();
await this.resolver.resolveChainedCallsViaConformance();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] chainedConformance: ${Date.now() - tChained}ms`);
// Same lifecycle for `this.<member>` callback registrations whose
// member is inherited from a supertype (#808).
const tDeferred = Date.now();
await this.resolver.resolveDeferredThisMemberRefs();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] deferredThisMember: ${Date.now() - tDeferred}ms`);
}
if (result.success) this.refreshNearDuplicates();
// Refresh planner stats + checkpoint the WAL after bulk writes.
// Off-thread (worker connection): on a multi-GB index this is minutes
// of IO, and inline it starved the #850 watchdog AFTER a fully
// successful index. Never load-bearing for correctness.
if (result.success && result.filesIndexed > 0) {
const tMaint = Date.now();
// Quiesce the valve first so its in-flight checkpoint and the
// maintenance checkpoint don't contend for the checkpointer lock
// (the loser would silently no-op and leave the WAL unfolded).
if (walValve) { walValve.stop(); await walValve.drain(); }
await this.db.runMaintenance();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] maintenance: ${Date.now() - tMaint}ms`);
}
// The orchestrator only sees extraction-phase counts; resolution and
// synthesizer edges (often >50% of the graph on JVM repos) come later.
// Recompute against the DB so the CLI summary reports the true totals.
if (result.success && result.filesIndexed > 0) {
const tCount = Date.now();
const after = this.queries.getNodeAndEdgeCount();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] count-recompute: ${Date.now() - tCount}ms`);
result.nodesCreated = after.nodes - before.nodes;
result.edgesCreated = after.edges - before.edges;
}
// Stamp the index with the engine that built it, so `codegraph status`
// and `codegraph upgrade` can recommend a re-index when the running
// engine produces richer extraction than the one on disk. Only on a
// real full index — a sync touches a subset, so it must NOT advance the
// extraction stamp (the bulk would still be stale). See extraction-version.ts.
if (result.success) this.stampPythonRootFingerprint();
if (result.success && result.filesIndexed > 0) {
try {
this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion);
this.queries.setMetadata('indexed_with_extraction_version', String(EXTRACTION_VERSION));
// ...and the commit whose tree it was built from, so change
// detection can ask git what has been COMMITTED since — the half
// `git status` structurally cannot answer. (#1829)
const head = extraction().getGitHeadSha(this.projectRoot);
if (head) this.queries.setMetadata(extraction().INDEXED_AT_COMMIT_KEY, head);
} catch { /* metadata is advisory — never fail an index over it */ }
}
if (result.success && result.filesErrored === 0 &&
(result.filesDiscovered === undefined || result.filesIndexed + result.filesSkipped >= result.filesDiscovered)) {
this.orchestrator.finishGitIndexState(gitState, true);
}
// Reconcile the scan's ground truth against what the pipeline
// accounted for. A shortfall means files were silently dropped
// (observed in the wild: a run under heavy load came up 37 files
// short with no error) — record it and tell the user, don't let the
// index pass as complete.
try {
if (!result.success) {
this.queries.setMetadata('index_state', 'failed');
} else {
const accounted = result.filesIndexed + result.filesSkipped + result.filesErrored;
const discovered = result.filesDiscovered;
const shortfall = discovered !== undefined ? discovered - accounted : 0;
if (discovered !== undefined && shortfall > 0) {
this.queries.setMetadata('index_state', 'partial');
this.queries.setMetadata('index_files_discovered', String(discovered));
this.queries.setMetadata('index_files_accounted', String(accounted));
result.errors.push({
message: `Index is missing ${shortfall} of ${discovered} discovered files (indexed ${result.filesIndexed}, skipped ${result.filesSkipped}, errored ${result.filesErrored}). The index is PARTIAL — re-run \`codegraph index\`.`,
severity: 'warning',
code: 'index_partial',
});
} else {
this.queries.setMetadata('index_state', 'complete');
if (discovered !== undefined) {
this.queries.setMetadata('index_files_discovered', String(discovered));
this.queries.setMetadata('index_files_accounted', String(accounted));
}
}
}
} catch { /* metadata is advisory — never fail an index over it */ }
// The orchestrator's duration covers extraction only; report the whole
// run, resolution and synthesis included.
result.durationMs = Date.now() - startedAt;
return result;
} finally {
// Restore the auto-checkpoint interval AFTER the fold-up above so the
// next ordinary write doesn't inherit a giant inline checkpoint. On
// the error path the WAL may still be large; correctness is unchanged
// (SQLite replays the WAL on the next open) and the follow-up write
// that folds it is the known cost of a failed run.
if (walValve) { walValve.stop(); await walValve.drain(); }
// Deterministic kernel-conn teardown while the valve is stopped and
// the pool is gone — a GC-timed rusqlite close could otherwise fire
// mid-shm-I/O in a LATER run (cross-build wal-index locks can't see
// each other → SIGBUS / overwritten WAL frames on the linux corpus).
this.resolver.closeKernel();
if (deferWal || restoreAutocheckpoint) {
try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ }
}
if (fastInit) {
// Back to the durable defaults; journal_mode=WAL folds the MEMORY
// journal state into a normal WAL-mode database file.
try {
this.db.getDb().pragma('synchronous = NORMAL');
this.db.getDb().pragma('journal_mode = WAL');
} catch { /* connection may be closing */ }
}
this.fileLock.release();
}
});
}
/**
* Index specific files
*
* Uses a mutex to prevent concurrent indexing operations.
*/
async indexFiles(filePaths: string[]): Promise<IndexResult> {
return this.indexMutex.withLock(async () => {
try {
this.fileLock.acquire();
} catch {
return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
}
try {
return this.orchestrator.indexFiles(filePaths);
} finally {
this.fileLock.release();
}
});
}
/**
* Sync with current file state (incremental update)
*
* Uses a mutex to prevent concurrent indexing operations.
*/
/** Indexed Python file paths — the before/after sets a sync diffs. */
private pythonFilePaths(): Set<string> {
return new Set(this.queries.getAllFilePaths().filter((f) => f.endsWith('.py')));
}
/** Record the Python package-root digest a later sync compares against. */
private stampPythonRootFingerprint(fingerprint = this.resolver.getPythonRootFingerprint()): void {
try {
this.queries.setMetadata(PYTHON_ROOTS_METADATA_KEY, fingerprint);
} catch { /* metadata is advisory */ }
}
async sync(options: IndexOptions = {}): Promise<SyncResult> {
if (options.requireComplete && process.env.CODEGRAPH_SYNC_RESYNTHESIS === '0') {
throw new Error('Complete reconciliation requires synthesized-edge refresh to be enabled.');
}
return this.indexMutex.withLock(async () => {
try {
this.fileLock.acquire();
} catch (err) {
throw new (sync().LockUnavailableError)(
`Sync could not acquire the file lock; retry when the index is available. ${err instanceof Error ? err.message : String(err)}`
);
}
const startedAt = Date.now();
// A full rebuild in another process (`codegraph index` → recreate)
// unlinks the database and creates a new file at the same path. A
// long-lived instance — the MCP daemon's watcher — would otherwise keep
// "syncing" into the dead inode, and nothing it wrote there is visible
// to anyone (#1902). Follow the path before writing (one stat), and
// widen a scoped sync to a full one: whatever the old handle absorbed
// since the rebuild is gone, so the new file has to be reconciled whole.
// If the reopen fails (the rebuild is mid-way), report lock contention
// so the watcher keeps its pending files and retries.
try {
this.reopenReplacedDatabase();
} catch (err) {
this.fileLock.release();
throw new (sync().LockUnavailableError)(
`Sync could not open the rebuilt index yet; retry when the rebuild finishes. ${err instanceof Error ? err.message : String(err)}`