diff --git a/lib/loading/ComponentsManagerBuilder.ts b/lib/loading/ComponentsManagerBuilder.ts index 6d84813..d952da9 100644 --- a/lib/loading/ComponentsManagerBuilder.ts +++ b/lib/loading/ComponentsManagerBuilder.ts @@ -20,6 +20,7 @@ import { ComponentRegistryFinalizer } from './ComponentRegistryFinalizer'; import { ConfigRegistry } from './ConfigRegistry'; import { ModuleStateBuilder } from './ModuleStateBuilder'; import type { IModuleState } from './ModuleStateBuilder'; +import { ModuleStateCache } from './ModuleStateCache'; /** * Builds {@link ComponentsManager}'s based on given options. @@ -32,6 +33,7 @@ export class ComponentsManagerBuilder { private readonly dumpErrorState: boolean; private readonly logger: Logger; private readonly moduleState?: IModuleState; + private readonly moduleStateCachePath?: string; private readonly skipContextValidation: boolean; private readonly typeChecking: boolean; private readonly remoteContextLookups: boolean; @@ -46,6 +48,7 @@ export class ComponentsManagerBuilder { this.dumpErrorState = options.dumpErrorState === undefined ? true : Boolean(options.dumpErrorState); this.logger = ComponentsManagerBuilder.createLogger(options.logLevel); this.moduleState = options.moduleState; + this.moduleStateCachePath = options.moduleStateCachePath; this.skipContextValidation = options.skipContextValidation === undefined ? true : Boolean(options.skipContextValidation); @@ -85,14 +88,27 @@ export class ComponentsManagerBuilder { */ public async build(): Promise> { // Initialize module state - let moduleState: IModuleState; - if (this.moduleState) { - moduleState = this.moduleState; - } else { + let moduleState: IModuleState | undefined = this.moduleState; + let moduleStateCache: ModuleStateCache | undefined; + if (!moduleState && this.moduleStateCachePath) { + moduleStateCache = new ModuleStateCache({ + path: this.moduleStateCachePath, + mainModulePath: this.mainModulePath, + logger: this.logger, + }); + moduleState = await moduleStateCache.load(); + if (moduleState) { + this.logger.info(`Loaded component discovery state from ${this.moduleStateCachePath}`); + } + } + if (!moduleState) { this.logger.info(`Initiating component discovery from ${this.mainModulePath}`); moduleState = await new ModuleStateBuilder(this.logger) .buildModuleState(require, this.mainModulePath); this.logger.info(`Discovered ${Object.keys(moduleState.componentModules).length} component packages within ${moduleState.nodeModulePaths.length} packages`); + if (moduleStateCache) { + await moduleStateCache.save(moduleState); + } } // Initialize object loader with built-in context @@ -212,6 +228,16 @@ export interface IComponentsManagerBuilderOptions { * Defaults to a newly created instances on the {@link mainModulePath}. */ moduleState?: IModuleState; + /** + * A file path to persist the module state to, to skip component discovery on + * subsequent invocations. Ignored when {@link moduleState} is provided. + * + * The persisted state is invalidated when the componentsjs version, or the + * modification time or size of the main module's package.json, lock files, or + * node_modules directory changes. In-place modifications deep inside + * node_modules are NOT detected; remove the cache file manually in that case. + */ + moduleStateCachePath?: string; /** * If JSON-LD context validation should be skipped. * Defaults to `true`. diff --git a/lib/loading/ModuleStateCache.ts b/lib/loading/ModuleStateCache.ts new file mode 100644 index 0000000..3b6128f --- /dev/null +++ b/lib/loading/ModuleStateCache.ts @@ -0,0 +1,128 @@ +import { promises as fs } from 'node:fs'; +import * as Path from 'node:path'; +import type { Logger } from 'winston'; + +// eslint-disable-next-line import/extensions +import packageJson from '../../package.json'; +import type { IModuleState } from './ModuleStateBuilder'; + +/** + * The version of the cache file format. + * Increment when the shape of the persisted data (or of {@link IModuleState}) changes. + */ +const CACHE_FORMAT_VERSION = 1; + +/** + * The files (relative to the main module path) whose modification time and size + * are included in the staleness fingerprint of a persisted module state. + */ +const FINGERPRINT_PATHS = [ + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'npm-shrinkwrap.json', + 'node_modules', +]; + +/** + * Persists an {@link IModuleState} (the result of component discovery over the + * dependency tree) to a file, so that subsequent invocations can skip the + * discovery scan entirely. + * + * Staleness handling: the cache stores a fingerprint of the cache format version, + * the componentsjs version, the main module path, and the modification time and + * size of the main module's package.json, lock files, and node_modules directory. + * A fingerprint mismatch (or any read/parse failure) makes {@link ModuleStateCache.load} + * return `undefined`, after which the caller is expected to run a fresh discovery + * scan and {@link ModuleStateCache.save} its result. + * + * This is a heuristic: package installations and removals touch a lock file and + * the node_modules directory, and are detected. In-place modifications deep + * inside node_modules (such as manually editing an installed package's component + * files) are NOT detected; in such cases the cache file must be removed manually + * (or the option not be used). + */ +export class ModuleStateCache { + private readonly path: string; + private readonly mainModulePath: string; + private readonly logger?: Logger; + + public constructor(options: IModuleStateCacheOptions) { + this.path = options.path; + this.mainModulePath = options.mainModulePath; + this.logger = options.logger; + } + + /** + * Compute the current staleness fingerprint for the main module path. + */ + public async fingerprint(): Promise { + const entries = await Promise.all(FINGERPRINT_PATHS.map(async(subPath) => { + try { + const stat = await fs.stat(Path.posix.join(this.mainModulePath, subPath)); + return [ subPath, stat.mtimeMs, stat.size ]; + } catch { + return [ subPath, null, null ]; + } + })); + return JSON.stringify([ CACHE_FORMAT_VERSION, packageJson.version, this.mainModulePath, entries ]); + } + + /** + * Load the persisted module state, if it exists and is fresh. + * @returns The module state, or `undefined` if there is no (fresh, readable) cache entry. + */ + public async load(): Promise { + let payload: any; + try { + payload = JSON.parse(await fs.readFile(this.path, 'utf8')); + } catch { + // No (readable) cache file + return; + } + if (!payload || typeof payload !== 'object' || payload.fingerprint !== await this.fingerprint()) { + if (this.logger) { + this.logger.info(`Ignoring stale module state cache at ${this.path}`); + } + return; + } + return payload.moduleState; + } + + /** + * Persist the given module state (best-effort: failures are logged, not thrown). + * @param moduleState A module state. + */ + public async save(moduleState: IModuleState): Promise { + try { + const payload = JSON.stringify({ + fingerprint: await this.fingerprint(), + moduleState, + }); + // Write-then-rename, so concurrent invocations never observe a partial cache file. + const temporaryPath = `${this.path}.${process.pid}.tmp`; + await fs.writeFile(temporaryPath, payload, 'utf8'); + await fs.rename(temporaryPath, this.path); + } catch (error: unknown) { + if (this.logger) { + this.logger.warn(`Failed to save module state cache to ${this.path}: ${( error).message}`); + } + } + } +} + +export interface IModuleStateCacheOptions { + /** + * The file path to persist the module state to. + */ + path: string; + /** + * Absolute path to the package root from which module resolution starts. + */ + mainModulePath: string; + /** + * An optional logger. + */ + logger?: Logger; +} diff --git a/test/unit/loading/ComponentsManagerBuilder-test.ts b/test/unit/loading/ComponentsManagerBuilder-test.ts index 55e147f..727fb4a 100644 --- a/test/unit/loading/ComponentsManagerBuilder-test.ts +++ b/test/unit/loading/ComponentsManagerBuilder-test.ts @@ -52,9 +52,64 @@ jest.mock('../../../lib/loading/ModuleStateBuilder', () => ({ }, })); +const moduleStateCache = { + load: jest.fn(), + save: jest.fn(), +}; +// eslint-disable-next-line jest/no-untyped-mock-factory +jest.mock('../../../lib/loading/ModuleStateCache', () => ({ + // eslint-disable-next-line object-shorthand + ModuleStateCache: function() { + return moduleStateCache; + }, +})); + describe('ComponentsManagerBuilder', () => { beforeEach(() => { jest.clearAllMocks(); + moduleStateCache.load.mockResolvedValue(undefined); + moduleStateCache.save.mockResolvedValue(undefined); + }); + + it('should build from a fresh module state cache', async() => { + moduleStateCache.load.mockResolvedValue(dummyModuleState); + const builder = new ComponentsManagerBuilder({ + mainModulePath, + moduleStateCachePath: '/tmp/module-state.json', + }); + const mgr = await builder.build(); + expect(mgr.moduleState).toBe(dummyModuleState); + expect(moduleStateCache.load).toHaveBeenCalledTimes(1); + expect(moduleStateCache.save).not.toHaveBeenCalled(); + }); + + it('should build and persist on a stale module state cache', async() => { + const builder = new ComponentsManagerBuilder({ + mainModulePath, + moduleStateCachePath: '/tmp/module-state.json', + }); + const mgr = await builder.build(); + expect(mgr.moduleState).toBe(dummyModuleState); + expect(moduleStateCache.load).toHaveBeenCalledTimes(1); + expect(moduleStateCache.save).toHaveBeenCalledTimes(1); + expect(moduleStateCache.save).toHaveBeenCalledWith(dummyModuleState); + }); + + it('should ignore the module state cache when a module state is provided', async() => { + const customModuleState = { + mainModulePath, + componentModules: {}, + nodeModulePaths: [], + }; + const builder = new ComponentsManagerBuilder({ + mainModulePath, + moduleState: customModuleState, + moduleStateCachePath: '/tmp/module-state.json', + }); + const mgr = await builder.build(); + expect(mgr.moduleState).toBe(customModuleState); + expect(moduleStateCache.load).not.toHaveBeenCalled(); + expect(moduleStateCache.save).not.toHaveBeenCalled(); }); it('should build with default options', async() => { diff --git a/test/unit/loading/ModuleStateCache-test.ts b/test/unit/loading/ModuleStateCache-test.ts new file mode 100644 index 0000000..66fa408 --- /dev/null +++ b/test/unit/loading/ModuleStateCache-test.ts @@ -0,0 +1,147 @@ +import { mocked } from 'jest-mock'; +import type { IModuleState } from '../../../lib/loading/ModuleStateBuilder'; +import { ModuleStateCache } from '../../../lib/loading/ModuleStateCache'; + +const fs = require('node:fs').promises; + +// eslint-disable-next-line jest/no-untyped-mock-factory +jest.mock('fs', () => ({ + promises: { + stat: jest.fn(), + readFile: jest.fn(), + writeFile: jest.fn(), + rename: jest.fn(), + }, +})); + +describe('ModuleStateCache', () => { + let logger: any; + let files: Record; + let fileContents: Record; + let written: Record; + let moduleState: IModuleState; + + beforeEach(() => { + jest.clearAllMocks(); + logger = { info: jest.fn(), warn: jest.fn() }; + files = { + '/main/package.json': { mtimeMs: 1_000, size: 10 }, + '/main/yarn.lock': { mtimeMs: 2_000, size: 20 }, + '/main/node_modules': { mtimeMs: 3_000, size: 30 }, + }; + fileContents = {}; + written = {}; + moduleState = { + mainModulePath: '/main', + nodeModulePaths: [ '/main' ], + packageJsons: { '/main': { name: 'main' }}, + }; + mocked(fs.stat).mockImplementation( (async(path: string) => { + if (!(path in files)) { + throw new Error(`File stat not found: ${path}`); + } + return files[path]; + })); + mocked(fs.readFile).mockImplementation( (async(path: string) => { + if (!(path in fileContents)) { + throw new Error(`File not found: ${path}`); + } + return fileContents[path]; + })); + mocked(fs.writeFile).mockImplementation( (async(path: string, contents: string) => { + written[path] = contents; + })); + mocked(fs.rename).mockImplementation( (async(from: string, to: string) => { + fileContents[to] = written[from]; + delete written[from]; + })); + }); + + function createCache(withLogger = true): ModuleStateCache { + return new ModuleStateCache({ + path: '/tmp/cache.json', + mainModulePath: '/main', + logger: withLogger ? logger : undefined, + }); + } + + describe('fingerprint', () => { + it('should include present and absent fingerprint files', async() => { + const fingerprint = JSON.parse(await createCache().fingerprint()); + const entries = Object.fromEntries(fingerprint[3].map((entry: any[]) => [ entry[0], entry.slice(1) ])); + expect(entries['package.json']).toEqual([ 1_000, 10 ]); + expect(entries['yarn.lock']).toEqual([ 2_000, 20 ]); + expect(entries.node_modules).toEqual([ 3_000, 30 ]); + expect(entries['package-lock.json']).toEqual([ null, null ]); + expect(entries['pnpm-lock.yaml']).toEqual([ null, null ]); + }); + + it('should change when a fingerprint file changes', async() => { + const before = await createCache().fingerprint(); + files['/main/yarn.lock'] = { mtimeMs: 2_001, size: 20 }; + await expect(createCache().fingerprint()).resolves.not.toEqual(before); + }); + }); + + describe('load', () => { + it('should return undefined without a cache file', async() => { + await expect(createCache().load()).resolves.toBeUndefined(); + }); + + it('should return undefined for a corrupt cache file', async() => { + fileContents['/tmp/cache.json'] = '{ corrupt'; + await expect(createCache().load()).resolves.toBeUndefined(); + }); + + it('should return undefined for a non-object cache file', async() => { + fileContents['/tmp/cache.json'] = 'null'; + await expect(createCache().load()).resolves.toBeUndefined(); + expect(logger.info).toHaveBeenCalledWith(`Ignoring stale module state cache at /tmp/cache.json`); + }); + + it('should return undefined for a stale fingerprint', async() => { + const cache = createCache(); + await cache.save(moduleState); + files['/main/node_modules'] = { mtimeMs: 4_000, size: 31 }; + await expect(cache.load()).resolves.toBeUndefined(); + expect(logger.info).toHaveBeenCalledWith(`Ignoring stale module state cache at /tmp/cache.json`); + }); + + it('should return undefined for a stale fingerprint without a logger', async() => { + const cache = createCache(false); + await cache.save(moduleState); + files['/main/node_modules'] = { mtimeMs: 4_000, size: 31 }; + await expect(cache.load()).resolves.toBeUndefined(); + }); + + it('should round-trip a saved module state', async() => { + const cache = createCache(); + await cache.save(moduleState); + await expect(cache.load()).resolves.toEqual(moduleState); + }); + }); + + describe('save', () => { + it('should write via a temporary file', async() => { + await createCache().save(moduleState); + expect(fs.writeFile).toHaveBeenCalledWith(`/tmp/cache.json.${process.pid}.tmp`, expect.any(String), 'utf8'); + expect(fs.rename).toHaveBeenCalledWith(`/tmp/cache.json.${process.pid}.tmp`, '/tmp/cache.json'); + }); + + it('should warn on write failures', async() => { + mocked(fs.writeFile).mockImplementation( (async() => { + throw new Error('Disk full'); + })); + await createCache().save(moduleState); + expect(logger.warn).toHaveBeenCalledWith(`Failed to save module state cache to /tmp/cache.json: Disk full`); + }); + + it('should ignore write failures without a logger', async() => { + mocked(fs.writeFile).mockImplementation( (async() => { + throw new Error('Disk full'); + })); + await createCache(false).save(moduleState); + expect(logger.warn).not.toHaveBeenCalled(); + }); + }); +});