From dbe7f1c8c87c4bfc958c06cb74f831b694c558a1 Mon Sep 17 00:00:00 2001 From: Oswald Rodrigues Date: Mon, 13 Apr 2026 15:50:28 +0530 Subject: [PATCH] changes to implement per model caching --- src/entities/model-metadata.entity.ts | 10 +++ src/guards/authentication.guard.ts | 1 + src/helpers/cache-key.helper.ts | 17 ++++ src/repository/solid-base.repository.ts | 58 ++++++++++++- .../seed-data/solid-core-metadata.json | 47 +++++++++++ src/services/cache-bootstrap.service.ts | 45 ++++++++++ src/services/crud.service.ts | 33 +++++++- src/services/solid-cache.service.ts | 83 +++++++++++++++++++ src/solid-core.module.ts | 5 ++ 9 files changed, 291 insertions(+), 8 deletions(-) create mode 100644 src/helpers/cache-key.helper.ts create mode 100644 src/services/cache-bootstrap.service.ts create mode 100644 src/services/solid-cache.service.ts diff --git a/src/entities/model-metadata.entity.ts b/src/entities/model-metadata.entity.ts index bb3e5309..a550b96b 100755 --- a/src/entities/model-metadata.entity.ts +++ b/src/entities/model-metadata.entity.ts @@ -57,6 +57,16 @@ export class ModelMetadata extends CommonEntity { @ManyToOne(() => FieldMetadata, {}) userKeyField: FieldMetadata; + @Column({ name: "cache_enabled", default: false }) + cacheEnabled: boolean; + + @Column({ name: "cache_strategy", default: "ttl", nullable: true }) + cacheStrategy: 'ttl' | 'onReboot'; + + // -1 = no expiry (cache forever). Only applies to 'ttl' strategy. + @Column({ name: "cache_ttl", default: -1, nullable: true }) + cacheTtl: number; + @Column({ default: false }) isSystem: boolean; diff --git a/src/guards/authentication.guard.ts b/src/guards/authentication.guard.ts index 3436e3ba..5b992b12 100755 --- a/src/guards/authentication.guard.ts +++ b/src/guards/authentication.guard.ts @@ -62,6 +62,7 @@ export class AuthenticationGuard implements CanActivate { AUTH_TYPE_KEY, [context.getHandler(), context.getClass()], ) ?? [AuthenticationGuard.defaultAuthType]; + const guards = authTypes.map((type) => this.authTypeGuardMap[type]).flat(); let error = new UnauthorizedException(); diff --git a/src/helpers/cache-key.helper.ts b/src/helpers/cache-key.helper.ts new file mode 100644 index 00000000..39fc2759 --- /dev/null +++ b/src/helpers/cache-key.helper.ts @@ -0,0 +1,17 @@ +import { createHash } from 'crypto'; +import { SelectQueryBuilder } from 'typeorm'; + +/** + * Builds a deterministic cache key from the model name and the fully resolved + * QueryBuilder SQL + bound parameters. + * + * Format: solidx:{modelName}:{sha256(sql::params)} + * + * Full SHA-256 (64 hex chars) is used intentionally — a collision would silently + * serve wrong data, so truncation is not acceptable. + */ +export function buildCacheKey(modelName: string, qb: SelectQueryBuilder): string { + const raw = qb.getSql() + '::' + JSON.stringify(qb.getParameters()); + const hash = createHash('sha256').update(raw).digest('hex'); + return `solidx:${modelName}:${hash}`; +} diff --git a/src/repository/solid-base.repository.ts b/src/repository/solid-base.repository.ts index 1d552d37..22614ab5 100644 --- a/src/repository/solid-base.repository.ts +++ b/src/repository/solid-base.repository.ts @@ -2,8 +2,10 @@ import { camelize } from '@angular-devkit/core/src/utils/strings'; import { Logger } from '@nestjs/common'; import { CommonEntity } from 'src/entities/common.entity'; import { ModelMetadata } from 'src/entities/model-metadata.entity'; +import { buildCacheKey } from 'src/helpers/cache-key.helper'; import { ActiveUserData } from 'src/interfaces/active-user-data.interface'; import { RequestContextService } from 'src/services/request-context.service'; +import { CacheConfig, SolidCacheService } from 'src/services/solid-cache.service'; import { DataSource, EntityNotFoundError, @@ -22,16 +24,64 @@ import { PickKeysByType } from 'typeorm/common/PickKeysByType'; export class SolidBaseRepository extends Repository { protected readonly logger: Logger; + // undefined = not yet loaded, null = model not found in metadata + private _cacheConfig: CacheConfig | null | undefined = undefined; + constructor( entity: EntityTarget, dataSource: DataSource, protected readonly requestContextService: RequestContextService | null, protected readonly securityRuleRepository: SecurityRuleRepository | null, + protected readonly solidCacheService?: SolidCacheService, ) { super(entity, dataSource.createEntityManager()); this.logger = new Logger(this.constructor.name); } + /** + * Loads cache config for this model once and memoizes it on the repo instance. + * Since repos are singletons, this results in a single DB query per model per + * app lifetime — all subsequent calls are served from the in-process field. + */ + private async getCacheConfig(): Promise { + if (this._cacheConfig !== undefined) return this._cacheConfig; + + const modelName = this.modelSingularName(); + const modelMeta = await this.manager.getRepository(ModelMetadata).findOne({ + where: { singularName: modelName }, + select: ['cacheEnabled', 'cacheStrategy', 'cacheTtl'], + }); + + this._cacheConfig = modelMeta + ? { cacheEnabled: modelMeta.cacheEnabled, cacheStrategy: modelMeta.cacheStrategy, cacheTtl: modelMeta.cacheTtl } + : null; + + return this._cacheConfig; + } + + /** + * Wraps a QueryBuilder execution with cache read-through. + * If caching is not configured or not enabled for this model, the executor + * is called directly with no overhead. + */ + private async executeWithCache( + qb: SelectQueryBuilder, + executor: () => Promise, + ): Promise { + if (!this.solidCacheService) return executor(); + + const config = await this.getCacheConfig(); + if (!config?.cacheEnabled) return executor(); + + const key = buildCacheKey(this.modelSingularName(), qb); + const cached = await this.solidCacheService.get(key); + if (cached !== null) return cached; + + const result = await executor(); + await this.solidCacheService.set(key, result, config.cacheStrategy, config.cacheTtl); + return result; + } + modelSingularName(): string { return camelize(this.metadata.name); } @@ -84,7 +134,7 @@ export class SolidBaseRepository extends Repository { if (options) qb.setFindOptions(options); // <- applies where, relations, select, order, etc. } - return qb.getOne(); + return this.executeWithCache(qb, () => qb.getOne()); } /** @@ -117,7 +167,7 @@ export class SolidBaseRepository extends Repository { qb.setFindOptions(options); } - return qb.getMany(); + return this.executeWithCache(qb, () => qb.getMany()); } /** @@ -132,7 +182,7 @@ export class SolidBaseRepository extends Repository { qb.setFindOptions(options); } - return qb.getManyAndCount(); + return this.executeWithCache(qb, () => qb.getManyAndCount()); } /** @@ -146,7 +196,7 @@ export class SolidBaseRepository extends Repository { qb.setFindOptions(options); } - return qb.getCount(); + return this.executeWithCache(qb, () => qb.getCount()); } /** diff --git a/src/seeders/seed-data/solid-core-metadata.json b/src/seeders/seed-data/solid-core-metadata.json index fb08e95f..cd5fae6c 100755 --- a/src/seeders/seed-data/solid-core-metadata.json +++ b/src/seeders/seed-data/solid-core-metadata.json @@ -343,6 +343,53 @@ "relationCascade": "cascade", "relationModelModuleName": "solid-core", "isSystem": true + }, + { + "name": "cacheEnabled", + "displayName": "Cache Enabled", + "type": "boolean", + "defaultValue": "false", + "required": false, + "unique": false, + "index": false, + "private": false, + "encrypt": false, + "columnName": "cache_enabled", + "isSystem": true + }, + { + "name": "cacheStrategy", + "displayName": "Cache Strategy", + "type": "selectionStatic", + "ormType": "varchar", + "length": 64, + "required": false, + "unique": false, + "index": false, + "private": false, + "encrypt": false, + "columnName": "cache_strategy", + "defaultValue": "ttl", + "selectionValueType": "string", + "selectionStaticValues": [ + "ttl:TTL", + "onReboot:On Reboot" + ], + "isSystem": true + }, + { + "name": "cacheTtl", + "displayName": "Cache TTL (seconds)", + "type": "int", + "ormType": "integer", + "defaultValue": "-1", + "required": false, + "unique": false, + "index": false, + "private": false, + "encrypt": false, + "columnName": "cache_ttl", + "isSystem": true } ] }, diff --git a/src/services/cache-bootstrap.service.ts b/src/services/cache-bootstrap.service.ts new file mode 100644 index 00000000..56cc358f --- /dev/null +++ b/src/services/cache-bootstrap.service.ts @@ -0,0 +1,45 @@ +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { ModelMetadata } from 'src/entities/model-metadata.entity'; +import { EntityManager } from 'typeorm'; +import { SolidCacheService } from './solid-cache.service'; + +@Injectable() +export class CacheBootstrapService implements OnApplicationBootstrap { + private readonly logger = new Logger(CacheBootstrapService.name); + + constructor( + @InjectEntityManager() + private readonly entityManager: EntityManager, + private readonly solidCacheService: SolidCacheService, + ) {} + + /** + * On every app boot, bust Redis keys for all models configured with the + * 'onReboot' cache strategy. This ensures stale data from the previous + * instance does not survive a restart. + * + * In-memory store is cleared automatically on restart — no action needed there. + */ + async onApplicationBootstrap(): Promise { + try { + const models = await this.entityManager.find(ModelMetadata, { + where: { cacheEnabled: true, cacheStrategy: 'onReboot' }, + select: ['singularName'], + }); + + if (models.length === 0) return; + + this.logger.log(`Busting onReboot cache for ${models.length} model(s)...`); + + await Promise.all( + models.map(m => this.solidCacheService.bustModel(m.singularName)), + ); + + this.logger.log('onReboot cache bust complete.'); + } catch (err) { + // Never block startup due to cache failures + this.logger.warn(`Cache bootstrap bust failed: ${err.message}`); + } + } +} diff --git a/src/services/crud.service.ts b/src/services/crud.service.ts index 94348f58..18565409 100755 --- a/src/services/crud.service.ts +++ b/src/services/crud.service.ts @@ -1,5 +1,7 @@ import { BadRequestException, NotFoundException } from "@nestjs/common"; import { DiscoveryService, ModuleRef } from "@nestjs/core"; +import { buildCacheKey } from "src/helpers/cache-key.helper"; +import { CacheConfig, SolidCacheService } from "src/services/solid-cache.service"; import { isArray } from "class-validator"; import { CommonEntity, SettingService, SolidBaseRepository, User } from "src"; import { ERROR_MESSAGES } from "src/constants/error-messages"; @@ -46,6 +48,7 @@ export class CRUDService { // Add two generic value i.e private _crudHelperService: CrudHelperService; private _discoveryService: DiscoveryService; private _settingService: SettingService; + private _solidCacheService: SolidCacheService; constructor( readonly entityManager: EntityManager, @@ -72,6 +75,24 @@ export class CRUDService { // Add two generic value i.e return this._settingService ??= this.moduleRef.get(SettingService, { strict: false }); } + protected get solidCacheService(): SolidCacheService { + return this._solidCacheService ??= this.moduleRef.get(SolidCacheService, { strict: false }); + } + + private async executeQbWithCache( + qb: SelectQueryBuilder, + executor: () => Promise, + config: CacheConfig, + ): Promise { + const key = buildCacheKey(this.modelName, qb); + const cached = await this.solidCacheService.get(key); + if (cached !== null) return cached; + + const result = await executor(); + await this.solidCacheService.set(key, result, config.cacheStrategy, config.cacheTtl); + return result; + } + async create(createDto: any, files: Express.Multer.File[] = [], solidRequestContext: any = {}): Promise { // This class will be extended by the generated service class i.e PersonService // The data required to identify the model and module name will be passed from the generate CrudService subclass @@ -466,7 +487,8 @@ export class CRUDService { // Add two generic value i.e let { limit, offset, populateMedia, populateGroup, groupFilter } = basicFilterDto; const populateUserIdFields = this.crudHelperService.extractUserIdFieldsFromPopulate(basicFilterDto.populate); - const { singularName, internationalisation, draftPublishWorkflow } = await this.loadModel(); + const { singularName, internationalisation, draftPublishWorkflow, cacheEnabled, cacheStrategy, cacheTtl } = await this.loadModel(); + const cacheConfig: CacheConfig = { cacheEnabled, cacheStrategy, cacheTtl }; // Check wheather user has update permission for model if (solidRequestContext.activeUser) { const hasPermission = this.crudHelperService.hasReadPermissionOnModel(solidRequestContext.activeUser, singularName); @@ -522,7 +544,7 @@ export class CRUDService { // Add two generic value i.e qb = (internationalisation && draftPublishWorkflow) ? this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef) : this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias); - const { meta, records } = await this.handleNonGroupFind(qb, populateUserIdFields, populateMedia, offset, limit, alias); + const { meta, records } = await this.handleNonGroupFind(qb, populateUserIdFields, populateMedia, offset, limit, alias, cacheConfig); return { meta, records, @@ -530,8 +552,11 @@ export class CRUDService { // Add two generic value i.e } } - private async handleNonGroupFind(qb: SelectQueryBuilder, populateUserIdFields: UserIdFields[], populateMedia: string[], offset: number, limit: number, alias: string) { - const [entities, count] = await qb.getManyAndCount(); + private async handleNonGroupFind(qb: SelectQueryBuilder, populateUserIdFields: UserIdFields[], populateMedia: string[], offset: number, limit: number, alias: string, cacheConfig?: CacheConfig) { + const execute = () => qb.getManyAndCount(); + const [entities, count] = cacheConfig?.cacheEnabled + ? await this.executeQbWithCache(qb, execute, cacheConfig) + : await execute(); // Populate the entity with the userId fields if (populateUserIdFields && populateUserIdFields.length > 0) { diff --git a/src/services/solid-cache.service.ts b/src/services/solid-cache.service.ts new file mode 100644 index 00000000..e921d29a --- /dev/null +++ b/src/services/solid-cache.service.ts @@ -0,0 +1,83 @@ +import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cache } from 'cache-manager'; +import { isRedisConfigured } from 'src/helpers/environment.helper'; + +export interface CacheConfig { + cacheEnabled: boolean; + cacheStrategy: 'ttl' | 'onReboot'; + cacheTtl: number; // seconds. -1 = no expiry. +} + +@Injectable() +export class SolidCacheService { + private readonly logger = new Logger(SolidCacheService.name); + + constructor( + @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, + private readonly configService: ConfigService, + ) {} + + async get(key: string): Promise { + try { + const value = await this.cacheManager.get(key); + return value ?? null; + } catch (err) { + this.logger.warn(`Cache get failed for key "${key}": ${err.message}`); + return null; + } + } + + async set(key: string, value: any, strategy: 'ttl' | 'onReboot', ttl: number): Promise { + try { + // onReboot and ttl=-1 both mean no expiry. + // NestJS cache-manager uses 0 to mean "no expiry". + const ttlMs = (strategy === 'onReboot' || ttl === -1) ? 0 : ttl * 1000; + await this.cacheManager.set(key, value, ttlMs); + } catch (err) { + // Cache failures must never break the main request path + this.logger.warn(`Cache set failed for key "${key}": ${err.message}`); + } + } + + /** + * Busts all cached entries for a given model. + * + * For Redis: uses SCAN + DEL to find and remove all keys matching solidx:{modelName}:* + * For in-memory: no-op — in-memory store is cleared automatically on app restart. + */ + async bustModel(modelName: string): Promise { + if (!isRedisConfigured(this.configService)) { + // In-memory store is wiped on restart — nothing to bust + return; + } + + try { + await this.scanAndDeleteRedis(`solidx:${modelName}:*`); + this.logger.log(`Busted Redis cache for model "${modelName}"`); + } catch (err) { + this.logger.warn(`Cache bust failed for model "${modelName}": ${err.message}`); + } + } + + private async scanAndDeleteRedis(pattern: string): Promise { + // cache-manager-redis-store exposes the underlying ioredis client + const client = (this.cacheManager.store as any).getClient(); + let cursor = '0'; + + do { + const [nextCursor, keys]: [string, string[]] = await client.scan( + cursor, + 'MATCH', + pattern, + 'COUNT', + 100, + ); + cursor = nextCursor; + if (keys.length > 0) { + await client.del(...keys); + } + } while (cursor !== '0'); + } +} diff --git a/src/solid-core.module.ts b/src/solid-core.module.ts index 0dbeeddf..7021493c 100755 --- a/src/solid-core.module.ts +++ b/src/solid-core.module.ts @@ -127,6 +127,8 @@ import { TwilioSmsQueueSubscriberRedis } from './jobs/redis/twilio-sms-subscribe import { UserRegistrationListener } from './listeners/user-registration.listener'; import { GoogleOauthStrategy } from './passport-strategies/google-oauth.strategy'; import { AuthenticationService } from './services/authentication.service'; +import { CacheBootstrapService } from './services/cache-bootstrap.service'; +import { SolidCacheService } from './services/solid-cache.service'; import { BcryptService } from './services/bcrypt.service'; import { UuidExternalIdEntityComputedFieldProvider } from './services/computed-fields/entity/uuid-externalid-entity-computed-field-provider.service'; import { UuidExternalIdComputedFieldProvider } from './services/computed-fields/uuid-external-id-computed-field-provider.service'; @@ -628,6 +630,8 @@ import { Entity } from 'typeorm'; SoftDeleteAwareEventSubscriber, AccessTokenGuard, AuthenticationService, + CacheBootstrapService, + SolidCacheService, GoogleAuthenticationController, RefreshTokenIdsStorageService, GoogleOauthStrategy, @@ -858,6 +862,7 @@ import { Entity } from 'typeorm'; SolidMicroserviceAdapter, UserService, SettingService, + SolidCacheService, ], }) export class SolidCoreModule implements NestModule {