From 77427b76458d082fbbb538edbafc20ca8ce522e3 Mon Sep 17 00:00:00 2001 From: Jesse Wright <63333554+jeswr@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:33:38 +0000 Subject: [PATCH] perf: Reduce and parallelize file system operations in the module state scan ModuleStateBuilder.buildModuleState walks the entire node_modules tree on every ComponentsManager build (1,381 modules for a Community Solid Server install). Three refinements, none of which change the resulting module state: 1. buildNodeModulePathsInner awaited the recursion for each non-scoped dependency sequentially, while scoped dependencies were already handled with Promise.all. Both now recurse in parallel. (The ignorePaths check-and-set is synchronous after the realpath await, so deduplication is unaffected.) 2. buildPackageJsons performed a stat (fileExists) followed by a readFile for every module; it now reads directly and treats a read failure as "no package.json", halving the file system operations of the hottest phase (1,381 -> 0 extra stats). Malformed JSON still throws exactly as before. 3. preprocessPackageJson probed components/components.jsonld, components/context.jsonld, components/ and config/ with four sequential stats per lsd:module package; the probes now run in parallel. A/B against the real Community Solid Server node_modules tree (1,381 modules, warm file system cache, 9 reps, 2-core shared box, alternating arms in the same run): before: median 355 ms (min 324) after: median 259-267 ms (min 236) ~1.35x, -90 ms per boot The resulting module state is identical in both arms (1,381 nodeModulePaths / 1,381 packageJsons / 168 componentModules / 169 contexts / 175 importPaths). The gain grows with cold file caches, where the parallel recursion overlaps disk latency instead of paying it serially. Co-Authored-By: Claude Fable 5 --- lib/loading/ModuleStateBuilder.ts | 68 +++++++++++++++++-------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/loading/ModuleStateBuilder.ts b/lib/loading/ModuleStateBuilder.ts index 30db231..2c169e6 100644 --- a/lib/loading/ModuleStateBuilder.ts +++ b/lib/loading/ModuleStateBuilder.ts @@ -115,7 +115,8 @@ export class ModuleStateBuilder { // Recursively handle all the Node modules of this valid Node module const dependenciesPath = Path.posix.join(path, 'node_modules'); - for (const dependency of await fs.readdir(dependenciesPath)) { + const dependencies = await fs.readdir(dependenciesPath); + await Promise.all(dependencies.map(async(dependency) => { // Ignore hidden folders, such as .bin if (!dependency.startsWith('.')) { const dependencyPath = Path.posix.join(dependenciesPath, dependency); @@ -131,7 +132,7 @@ export class ModuleStateBuilder { await this.buildNodeModulePathsInner(dependencyPath, nodeModulePaths, ignorePaths); } } - } + })); } } catch { // Ignore invalid paths @@ -155,9 +156,16 @@ export class ModuleStateBuilder { const packageJsons: Record = {}; await Promise.all(nodeModulePaths.map(async(modulePath) => { const path = Path.posix.join(modulePath, 'package.json'); - if (await this.fileExists(path)) { - packageJsons[modulePath] = JSON.parse(await fs.readFile(path, 'utf8')); + // Try the read directly instead of checking existence first, + // which halves the number of file system operations. + let contents: string; + try { + contents = await fs.readFile(path, 'utf8'); + } catch { + // Ignore modules without a package.json file + return; } + packageJsons[modulePath] = JSON.parse(contents); })); return packageJsons; } @@ -181,37 +189,37 @@ export class ModuleStateBuilder { if (packageJson['lsd:module'] === true) { packageJson['lsd:module'] = `https://linkedsoftwaredependencies.org/bundles/npm/${packageJson.name}`; const basePath = packageJson['lsd:basePath'] || ''; - try { - if ((await fs.stat(Path.posix.join(packagePath, basePath, 'components/components.jsonld'))).isFile()) { - packageJson['lsd:components'] = `${basePath}components/components.jsonld`; - } - } catch { - // Ignore errors - } const baseIri = `${packageJson['lsd:module']}/^${semverMajor(packageJson.version)}.0.0/`; - try { - if ((await fs.stat(Path.posix.join(packagePath, basePath, 'components/context.jsonld'))).isFile()) { - packageJson['lsd:contexts'] = { - [`${baseIri}components/context.jsonld`]: `${basePath}components/context.jsonld`, - }; + // Probe the file system in parallel instead of sequentially. + const probe = async(subPath: string, file: boolean): Promise => { + try { + const stat = await fs.stat(Path.posix.join(packagePath, basePath, subPath)); + return file ? stat.isFile() : stat.isDirectory(); + } catch { + // Ignore errors + return false; } - } catch { - // Ignore errors + }; + const [ hasComponents, hasContext, hasComponentsDir, hasConfigDir ] = await Promise.all([ + probe('components/components.jsonld', true), + probe('components/context.jsonld', true), + probe('components', false), + probe('config', false), + ]); + if (hasComponents) { + packageJson['lsd:components'] = `${basePath}components/components.jsonld`; + } + if (hasContext) { + packageJson['lsd:contexts'] = { + [`${baseIri}components/context.jsonld`]: `${basePath}components/context.jsonld`, + }; } packageJson['lsd:importPaths'] = {}; - try { - if ((await fs.stat(Path.posix.join(packagePath, basePath, 'components'))).isDirectory()) { - packageJson['lsd:importPaths'][`${baseIri}components/`] = `${basePath}components/`; - } - } catch { - // Ignore errors + if (hasComponentsDir) { + packageJson['lsd:importPaths'][`${baseIri}components/`] = `${basePath}components/`; } - try { - if ((await fs.stat(Path.posix.join(packagePath, basePath, 'config'))).isDirectory()) { - packageJson['lsd:importPaths'][`${baseIri}config/`] = `${basePath}config/`; - } - } catch { - // Ignore errors + if (hasConfigDir) { + packageJson['lsd:importPaths'][`${baseIri}config/`] = `${basePath}config/`; } return true; }