feat: Add opt-in module state persistence for warm starts (scan 416 ms -> load 36 ms)

lib/loading/ComponentsManagerBuilder.ts

@@ -20,6 +20,7 @@ import { ComponentRegistryFinalizer } from './ComponentRegistryFinalizer';
 import { ConfigRegistry } from './ConfigRegistry';
 import { ModuleStateBuilder } from './ModuleStateBuilder';
 import type { IModuleState } from './ModuleStateBuilder';
+import { ModuleStateCache } from './ModuleStateCache';
 
 /**
  * Builds {@link ComponentsManager}'s based on given options.
@@ -32,6 +33,7 @@ export class ComponentsManagerBuilder<TInstance = any> {
   private readonly dumpErrorState: boolean;
   private readonly logger: Logger;
   private readonly moduleState?: IModuleState;
+  private readonly moduleStateCachePath?: string;
   private readonly skipContextValidation: boolean;
   private readonly typeChecking: boolean;
   private readonly remoteContextLookups: boolean;
@@ -46,6 +48,7 @@ export class ComponentsManagerBuilder<TInstance = any> {
     this.dumpErrorState = options.dumpErrorState === undefined ? true : Boolean(options.dumpErrorState);
     this.logger = ComponentsManagerBuilder.createLogger(options.logLevel);
     this.moduleState = options.moduleState;
+    this.moduleStateCachePath = options.moduleStateCachePath;
     this.skipContextValidation = options.skipContextValidation === undefined ?
       true :
       Boolean(options.skipContextValidation);
@@ -85,14 +88,27 @@ export class ComponentsManagerBuilder<TInstance = any> {
    */
   public async build(): Promise<ComponentsManager<TInstance>> {
     // Initialize module state
-    let moduleState: IModuleState;
-    if (this.moduleState) {
-      moduleState = this.moduleState;
-    } else {
+    let moduleState: IModuleState | undefined = this.moduleState;
+    let moduleStateCache: ModuleStateCache | undefined;
+    if (!moduleState && this.moduleStateCachePath) {
+      moduleStateCache = new ModuleStateCache({
+        path: this.moduleStateCachePath,
+        mainModulePath: this.mainModulePath,
+        logger: this.logger,
+      });
+      moduleState = await moduleStateCache.load();
+      if (moduleState) {
+        this.logger.info(`Loaded component discovery state from ${this.moduleStateCachePath}`);
+      }
+    }
+    if (!moduleState) {
       this.logger.info(`Initiating component discovery from ${this.mainModulePath}`);
       moduleState = await new ModuleStateBuilder(this.logger)
         .buildModuleState(require, this.mainModulePath);
       this.logger.info(`Discovered ${Object.keys(moduleState.componentModules).length} component packages within ${moduleState.nodeModulePaths.length} packages`);
+      if (moduleStateCache) {
+        await moduleStateCache.save(moduleState);
+      }
     }
 
     // Initialize object loader with built-in context
@@ -212,6 +228,16 @@ export interface IComponentsManagerBuilderOptions<TInstance> {
    * Defaults to a newly created instances on the {@link mainModulePath}.
    */
   moduleState?: IModuleState;
+  /**
+   * A file path to persist the module state to, to skip component discovery on
+   * subsequent invocations. Ignored when {@link moduleState} is provided.
+   *
+   * The persisted state is invalidated when the componentsjs version, or the
+   * modification time or size of the main module's package.json, lock files, or
+   * node_modules directory changes. In-place modifications deep inside
+   * node_modules are NOT detected; remove the cache file manually in that case.
+   */
+  moduleStateCachePath?: string;
   /**
    * If JSON-LD context validation should be skipped.
    * Defaults to `true`.

lib/loading/ModuleStateCache.ts

@@ -0,0 +1,128 @@
+import { promises as fs } from 'node:fs';
+import * as Path from 'node:path';
+import type { Logger } from 'winston';
+
+// eslint-disable-next-line import/extensions
+import packageJson from '../../package.json';
+import type { IModuleState } from './ModuleStateBuilder';
+
+/**
+ * The version of the cache file format.
+ * Increment when the shape of the persisted data (or of {@link IModuleState}) changes.
+ */
+const CACHE_FORMAT_VERSION = 1;
+
+/**
+ * The files (relative to the main module path) whose modification time and size
+ * are included in the staleness fingerprint of a persisted module state.
+ */
+const FINGERPRINT_PATHS = [
+  'package.json',
+  'package-lock.json',
+  'yarn.lock',
+  'pnpm-lock.yaml',
+  'npm-shrinkwrap.json',
+  'node_modules',
+];
+
+/**
+ * Persists an {@link IModuleState} (the result of component discovery over the
+ * dependency tree) to a file, so that subsequent invocations can skip the
+ * discovery scan entirely.
+ *
+ * Staleness handling: the cache stores a fingerprint of the cache format version,
+ * the componentsjs version, the main module path, and the modification time and
+ * size of the main module's package.json, lock files, and node_modules directory.
+ * A fingerprint mismatch (or any read/parse failure) makes {@link ModuleStateCache.load}
+ * return `undefined`, after which the caller is expected to run a fresh discovery
+ * scan and {@link ModuleStateCache.save} its result.
+ *
+ * This is a heuristic: package installations and removals touch a lock file and
+ * the node_modules directory, and are detected. In-place modifications deep
+ * inside node_modules (such as manually editing an installed package's component
+ * files) are NOT detected; in such cases the cache file must be removed manually
+ * (or the option not be used).
+ */
+export class ModuleStateCache {
+  private readonly path: string;
+  private readonly mainModulePath: string;
+  private readonly logger?: Logger;
+
+  public constructor(options: IModuleStateCacheOptions) {
+    this.path = options.path;
+    this.mainModulePath = options.mainModulePath;
+    this.logger = options.logger;
+  }
+
+  /**
+   * Compute the current staleness fingerprint for the main module path.
+   */
+  public async fingerprint(): Promise<string> {
+    const entries = await Promise.all(FINGERPRINT_PATHS.map(async(subPath) => {
+      try {
+        const stat = await fs.stat(Path.posix.join(this.mainModulePath, subPath));
+        return [ subPath, stat.mtimeMs, stat.size ];
+      } catch {
+        return [ subPath, null, null ];
+      }
+    }));
+    return JSON.stringify([ CACHE_FORMAT_VERSION, packageJson.version, this.mainModulePath, entries ]);
+  }
+
+  /**
+   * Load the persisted module state, if it exists and is fresh.
+   * @returns The module state, or `undefined` if there is no (fresh, readable) cache entry.
+   */
+  public async load(): Promise<IModuleState | undefined> {
+    let payload: any;
+    try {
+      payload = JSON.parse(await fs.readFile(this.path, 'utf8'));
+    } catch {
+      // No (readable) cache file
+      return;
+    }
+    if (!payload || typeof payload !== 'object' || payload.fingerprint !== await this.fingerprint()) {
+      if (this.logger) {
+        this.logger.info(`Ignoring stale module state cache at ${this.path}`);
+      }
+      return;
+    }
+    return payload.moduleState;
+  }
+
+  /**
+   * Persist the given module state (best-effort: failures are logged, not thrown).
+   * @param moduleState A module state.
+   */
+  public async save(moduleState: IModuleState): Promise<void> {
+    try {
+      const payload = JSON.stringify({
+        fingerprint: await this.fingerprint(),
+        moduleState,
+      });
+      // Write-then-rename, so concurrent invocations never observe a partial cache file.
+      const temporaryPath = `${this.path}.${process.pid}.tmp`;
+      await fs.writeFile(temporaryPath, payload, 'utf8');
+      await fs.rename(temporaryPath, this.path);
+    } catch (error: unknown) {
+      if (this.logger) {
+        this.logger.warn(`Failed to save module state cache to ${this.path}: ${(<Error> error).message}`);
+      }
+    }
+  }
+}
+
+export interface IModuleStateCacheOptions {
+  /**
+   * The file path to persist the module state to.
+   */
+  path: string;
+  /**
+   * Absolute path to the package root from which module resolution starts.
+   */
+  mainModulePath: string;
+  /**
+   * An optional logger.
+   */
+  logger?: Logger;
+}

test/unit/loading/ComponentsManagerBuilder-test.ts

@@ -52,9 +52,64 @@ jest.mock('../../../lib/loading/ModuleStateBuilder', () => ({
   },
 }));
 
+const moduleStateCache = {
+  load: jest.fn(),
+  save: jest.fn(),
+};
+// eslint-disable-next-line jest/no-untyped-mock-factory
+jest.mock('../../../lib/loading/ModuleStateCache', () => ({
+  // eslint-disable-next-line object-shorthand
+  ModuleStateCache: function() {
+    return moduleStateCache;
+  },
+}));
+
 describe('ComponentsManagerBuilder', () => {
   beforeEach(() => {
     jest.clearAllMocks();
+    moduleStateCache.load.mockResolvedValue(undefined);
+    moduleStateCache.save.mockResolvedValue(undefined);
+  });
+
+  it('should build from a fresh module state cache', async() => {
+    moduleStateCache.load.mockResolvedValue(dummyModuleState);
+    const builder = new ComponentsManagerBuilder({
+      mainModulePath,
+      moduleStateCachePath: '/tmp/module-state.json',
+    });
+    const mgr = await builder.build();
+    expect(mgr.moduleState).toBe(dummyModuleState);
+    expect(moduleStateCache.load).toHaveBeenCalledTimes(1);
+    expect(moduleStateCache.save).not.toHaveBeenCalled();
+  });
+
+  it('should build and persist on a stale module state cache', async() => {
+    const builder = new ComponentsManagerBuilder({
+      mainModulePath,
+      moduleStateCachePath: '/tmp/module-state.json',
+    });
+    const mgr = await builder.build();
+    expect(mgr.moduleState).toBe(dummyModuleState);
+    expect(moduleStateCache.load).toHaveBeenCalledTimes(1);
+    expect(moduleStateCache.save).toHaveBeenCalledTimes(1);
+    expect(moduleStateCache.save).toHaveBeenCalledWith(dummyModuleState);
+  });
+
+  it('should ignore the module state cache when a module state is provided', async() => {
+    const customModuleState = <any> {
+      mainModulePath,
+      componentModules: {},
+      nodeModulePaths: [],
+    };
+    const builder = new ComponentsManagerBuilder({
+      mainModulePath,
+      moduleState: customModuleState,
+      moduleStateCachePath: '/tmp/module-state.json',
+    });
+    const mgr = await builder.build();
+    expect(mgr.moduleState).toBe(customModuleState);
+    expect(moduleStateCache.load).not.toHaveBeenCalled();
+    expect(moduleStateCache.save).not.toHaveBeenCalled();
   });
 
   it('should build with default options', async() => {