From 043f493c6145fcf938ba6f8b105301e1badf9766 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:14:02 +0300 Subject: [PATCH 01/66] feat: wip implement delete layer --- .vscode/settings.json | 2 +- config/custom-environment-variables.json | 37 +- config/default.json | 52 ++- helm/templates/configmap.yaml | 8 + helm/templates/deployment.yaml | 6 +- helm/values.yaml | 5 + package-lock.json | 8 +- package.json | 2 +- src/cleaner/errors/errors.ts | 4 +- src/cleaner/errors/index.ts | 2 +- .../storageProviders/fsStorageProvider.ts | 138 ++++++-- .../storageProviders/iStorageProvider.ts | 21 +- src/cleaner/storageProviders/index.ts | 6 +- .../storageProviders/s3StorageProvider.ts | 208 +++++++++-- .../deleteStoredResourcesStrategy.ts | 66 ++++ src/cleaner/strategies/index.ts | 3 +- src/cleaner/strategies/strategyFactory.ts | 15 +- .../strategies/tilesDeletionStrategy.ts | 40 ++- src/cleaner/utils/chunk.ts | 5 + src/cleaner/utils/index.ts | 1 + src/common/dependencyRegistration.ts | 2 + src/containerConfig.ts | 69 +++- tests/helpers/mocks.ts | 43 ++- .../fsStorageProvider.spec.ts | 141 +++++++- .../s3StorageProvider.spec.ts | 332 +++++++++++++++++- .../deleteStoredResourcesStrategy.spec.ts | 135 +++++++ tests/strategyFactory.spec.ts | 59 ++-- tests/tilesDeletionStrategy.spec.ts | 42 ++- 28 files changed, 1250 insertions(+), 202 deletions(-) create mode 100644 src/cleaner/strategies/deleteStoredResourcesStrategy.ts create mode 100644 src/cleaner/utils/chunk.ts create mode 100644 tests/strategies/deleteStoredResourcesStrategy.spec.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 009aae3..8644b0a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "typescript.tsdk": "node_modules/typescript/lib", + "js/ts.tsdk.path": "node_modules/typescript/lib", "files.eol": "\n", "editor.defaultFormatter": "esbenp.prettier-vscode", "files.insertFinalNewline": true diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 11aaa93..47787f9 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -73,19 +73,28 @@ "__format": "boolean" } }, - "s3": { - "endpoint": "S3_ENDPOINT", - "accessKeyId": "S3_ACCESS_KEY_ID", - "secretAccessKey": "S3_SECRET_ACCESS_KEY", - "sslEnabled": { - "__name": "S3_SSL_ENABLED", - "__format": "boolean" + "storage": { + "cleanupStorageProviders": { + "__name": "CLEANUP_STORAGE_PROVIDERS", + "__format": "json" }, - "forcePathStyle": { - "__name": "S3_FORCE_PATH_STYLE", - "__format": "boolean" + "s3": { + "endpoint": "S3_ENDPOINT", + "accessKeyId": "S3_ACCESS_KEY_ID", + "secretAccessKey": "S3_SECRET_ACCESS_KEY", + "sslEnabled": { + "__name": "S3_SSL_ENABLED", + "__format": "boolean" + }, + "forcePathStyle": { + "__name": "S3_FORCE_PATH_STYLE", + "__format": "boolean" + }, + "region": "S3_REGION" }, - "region": "S3_REGION" + "fs": { + "basePath": "FS_BASE_PATH" + } }, "strategies": { "tilesDeletion": { @@ -103,6 +112,12 @@ "__name": "TILES_DELETION_FAILURE_SAMPLE_SIZE", "__format": "number" } + }, + "storedResourcesDeletion": { + "failureSampleSize": { + "__name": "STORED_RESOURCES_DELETION_FAILURE_SAMPLE_SIZE", + "__format": "number" + } } } } diff --git a/config/default.json b/config/default.json index 901d821..d2be632 100644 --- a/config/default.json +++ b/config/default.json @@ -9,9 +9,7 @@ "level": "info", "prettyPrint": false, "opentelemetryOptions": { - "enabled": false, - "url": "", - "resourceAttributes": {} + "enabled": false } } }, @@ -28,18 +26,26 @@ { "job": "Ingestion_Swap_Update", "task": "tiles-deletion" + }, + { + "job": "Delete_Layer", + "task": "tiles-deletion" + }, + { + "job": "Delete_Layer", + "task": "artifacts-deletion" } ] } }, "queue": { - "jobManagerBaseUrl": "http://localhost:8080", - "heartbeatBaseUrl": "http://localhost:8081", + "jobManagerBaseUrl": "http://localhost:8081", + "heartbeatBaseUrl": "http://localhost:8082", "heartbeatIntervalMs": 1000, "dequeueIntervalMs": 3000 }, "servicesUrl": { - "jobTracker": "http://localhost:8082" + "jobTracker": "http://localhost:8083" }, "disableHttpClientLogs": false, "jobDefinitions": { @@ -49,12 +55,23 @@ }, "swapUpdate": { "type": "Ingestion_Swap_Update" + }, + "deleteLayer": { + "type": "Delete_Layer" } }, "tasks": { "tilesDeletion": { "type": "tiles-deletion", "maxAttempts": 3 + }, + "layerDeletion": { + "type": "tiles-deletion", + "maxAttempts": 3 + }, + "artifactsDeletion": { + "type": "artifacts-deletion", + "maxAttempts": 3 } } }, @@ -63,13 +80,19 @@ "delay": "exponential", "shouldResetTimeout": true }, - "s3": { - "endpoint": "http://localhost:9000", - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin", - "sslEnabled": false, - "forcePathStyle": true, - "region": "us-east-1" + "storage": { + "cleanupStorageProviders": ["FS", "S3"], + "s3": { + "endpoint": "http://localhost:9000", + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin", + "sslEnabled": false, + "forcePathStyle": true, + "region": "us-east-1" + }, + "fs": { + "basePath": "/tiles" + } }, "strategies": { "tilesDeletion": { @@ -78,6 +101,9 @@ "batchSize": 1000, "concurrency": 10, "failureSampleSize": 3 + }, + "storedResourcesDeletion": { + "failureSampleSize": 3 } } } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 855dbaa..8051b0f 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -54,10 +54,15 @@ data: {{- with .Values.env.jobnik.worker }} JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }} {{- end }} + {{- if has "S3" $storage.cleanupStorageProviders }} S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }} S3_FORCE_PATH_STYLE: {{ $s3.forcePathStyle | default false | quote }} S3_SSL_ENABLED: {{ $s3.sslEnabled | default false | quote }} S3_REGION: {{ $s3.region | default "us-east-1" | quote }} + {{- end }} + {{- if has "FS" $storage.cleanupStorageProviders }} + FS_BASE_PATH: {{ $tilesFSBasePath | quote }} + {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} TILES_DELETION_FS_BASE_PATH: {{ $tilesFSBasePath | quote }} {{- with .Values.env.strategies.tilesDeletion }} @@ -65,4 +70,7 @@ data: TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }} TILES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }} {{- end }} + {{- with .Values.env.strategies.storedResourcesDeletion }} + STORED_RESOURCES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }} + {{- end }} {{- end }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index ec4d9ff..0b8be5f 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -54,10 +54,10 @@ spec: imagePullPolicy: {{ .pullPolicy | default "IfNotPresent" }} {{- end }} {{- if .Values.command }} - command: + command: {{- toYaml .Values.command | nindent 12 }} {{- if .Values.args }} - args: + args: {{- toYaml .Values.args | nindent 12 }} {{- end }} {{- end }} @@ -122,7 +122,7 @@ spec: httpGet: path: {{ .Values.readinessProbe.path }} port: {{ .Values.env.targetPort }} - {{- end }} + {{- end }} {{- if .Values.resources.enabled }} resources: {{- toYaml .Values.resources.value | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index be16b49..3eca562 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -13,6 +13,9 @@ serviceUrls: jobTracker: "" storage: + cleanupStorageProviders: + - FS + - S3 s3: endpointUrl: "" forcePathStyle: true @@ -128,6 +131,8 @@ env: batchSize: 1000 concurrency: 10 failureSampleSize: 3 + storedResourcesDeletion: + failureSampleSize: 3 resources: enabled: true diff --git a/package-lock.json b/package-lock.json index 4d16ca5..915d6fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.3.0-alpha.1", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", @@ -2543,9 +2543,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "8.1.0-alpha.3", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.1.0-alpha.3.tgz", - "integrity": "sha512-igF8yhnSeXaUdpx6gW3C5gi4AM9J56jVwgqhh7+Zv0mYb/yBhxv0xsv4Mhyv+41UpFCVduYf90DJZRrY29kLpQ==", + "version": "8.3.0-alpha.1", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.1.tgz", + "integrity": "sha512-KP/n3ZPcG7ULHrtX3h9fMOTNnF5Qkos1N9ZPjawi/5yiDHit9S5V9sFI4Tgv5aMchgSS0zXu5gL0ybA4/CliYQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index d71fedb..b6e4ebb 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.3.0-alpha.1", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", diff --git a/src/cleaner/errors/errors.ts b/src/cleaner/errors/errors.ts index a3efe73..5c10557 100644 --- a/src/cleaner/errors/errors.ts +++ b/src/cleaner/errors/errors.ts @@ -89,8 +89,8 @@ export class ValidationError extends UnrecoverableError { * This is always unrecoverable since the strategy won't appear on retry. */ export class StrategyNotFoundError extends UnrecoverableError { - public constructor(taskType: string) { - super(`No strategy registered for task type: ${taskType}`); + public constructor({ jobType, taskType }: { jobType: string; taskType: string }) { + super(`No strategy registered for job type: ${jobType} and task type: ${taskType}`); this.name = StrategyNotFoundError.name; Error.captureStackTrace(this, this.constructor); } diff --git a/src/cleaner/errors/index.ts b/src/cleaner/errors/index.ts index 0315ef3..330ee2f 100644 --- a/src/cleaner/errors/index.ts +++ b/src/cleaner/errors/index.ts @@ -1,2 +1,2 @@ -export { toError, describeError, RecoverableError, UnrecoverableError, ConfigurationError, ValidationError, StrategyNotFoundError } from './errors'; export { ErrorHandler } from './errorHandler'; +export { ConfigurationError, describeError, RecoverableError, StrategyNotFoundError, toError, UnrecoverableError, ValidationError } from './errors'; diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 87734d5..b8f1a35 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -1,11 +1,33 @@ -import { join } from 'node:path'; -import { stat, unlink, rmdir } from 'node:fs/promises'; +import { accessSync, constants, statSync } from 'node:fs'; +import { rm, rmdir, stat, unlink } from 'node:fs/promises'; +import { join, resolve, sep } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; -import { describeError } from '../errors'; -import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; +import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import type { ConfigType } from '@src/common/config'; +import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; +import { getChunk } from '../utils'; -export class FsStorageProvider implements IStorageProvider { - public constructor(private readonly logger: Logger) {} +const FS_MAX_DELETE_BATCH = 1000; + +type FSStorageProviderType = Extract; + +export interface FsConfig { + basePath: string; +} + +export class FsStorageProvider implements IStorageProvider<'FS'> { + private readonly basePath: string; + + public constructor( + private readonly config: ConfigType, + private readonly logger: Logger + ) { + const fsConfig = this.config.get('storage.fs') as unknown as FsConfig; + this.basePath = this.resolveAbsolutePath(fsConfig.basePath); + this.canDeleteFromFolder(this.basePath); + this.logger.debug(`Using ${this.basePath} as base path for FS`); + } public async targetExists(storageTarget: string, relativePath: string): Promise { try { @@ -18,26 +40,24 @@ export class FsStorageProvider implements IStorageProvider { } public async delete(paths: string[], storageTarget: string): Promise { - if (paths.length === 0) { - return []; - } - this.logger.info({ msg: 'Deleting files from filesystem', basePath: storageTarget, count: paths.length }); - const results = await Promise.allSettled( - paths.map(async (relativePath) => { - await unlink(join(storageTarget, relativePath)); - }) - ); - const failures: DeleteFailure[] = []; - for (const [idx, result] of results.entries()) { - if (result.status === 'rejected') { - const relativePath = paths[idx]!; - const error: unknown = result.reason; - const reason = describeError(error); - this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); - failures.push({ path: relativePath, reason }); + for (const relativePaths of getChunk(paths, FS_MAX_DELETE_BATCH)) { + const results = await Promise.allSettled( + relativePaths.map(async (relativePath) => { + await unlink(join(storageTarget, relativePath)); + }) + ); + + for (const [idx, result] of results.entries()) { + if (result.status === 'rejected') { + const relativePath = relativePaths[idx]!; + const error: unknown = result.reason; + const reason = describeError(error); + this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); + failures.push({ path: relativePath, reason }); + } } } @@ -46,6 +66,67 @@ export class FsStorageProvider implements IStorageProvider { return failures; } + public async deleteResources({ + paths, + }: Extract): Promise { + this.logger.info({ msg: 'Deleting multiple directories/files from filesystem', basePath: this.basePath, count: paths.length }); + + // prevent path traversal (i.e. accessing folders above root folder) + if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); + if (!paths.every((path) => this.resolveAbsolutePath(join(this.basePath, path)) !== this.basePath)) + throw new UnrecoverableError(`Cannot delete base path itself`); + + const failures: DeleteFailure[] = []; + for (const relativePaths of getChunk(paths, FS_MAX_DELETE_BATCH)) { + const results = await Promise.allSettled( + relativePaths.map(async (relativePath) => { + await rm(join(this.basePath, relativePath), { recursive: true, force: true }); + }) + ); + + for (const [idx, result] of results.entries()) { + if (result.status === 'rejected') { + const fullPath = join(this.basePath, relativePaths[idx]!); + const reason = describeError(result.reason); + this.logger.error({ msg: 'Failed to delete layer directory', fullPath, reason, err: result.reason }); + failures.push({ path: fullPath, reason }); + } + } + } + + return { failures }; + } + + private canDeleteFromFolder(path: string): void { + try { + accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); + } catch (err) { + if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { + throw new ConfigurationError(`FS path does not exist: ${path}`); + } else if (err instanceof Error && 'code' in err && (err.code === 'EACCES' || err.code === 'EPERM')) { + throw new ConfigurationError(`FS path permission denied for path: ${path}`); + } else { + throw new ConfigurationError(`An unexpected error occurred on FS path accessibility check: ${describeError(err)}`); + } + } + + try { + const pathStat = statSync(path); + if (!pathStat.isDirectory()) { + throw new ConfigurationError(`FS path exists but it is a file, not a directory: ${path}`); + } + } catch (err) { + throw new ConfigurationError(`An unexpected error occurred on FS info check: ${describeError(err)}`); + } + } + + private checkPathTraversal(paths: string[]): boolean { + return paths.every((path) => { + const absolutePath = this.resolveAbsolutePath(join(this.basePath, path)); + return absolutePath.startsWith(this.basePath); + }); + } + // Attempts to remove any directories that became empty after file deletion. // Strategy: collect every ancestor directory of every deleted file, grouped by // depth relative to the file (levelIdx 0 = direct parent, 1 = grandparent, …). @@ -88,4 +169,15 @@ export class FsStorageProvider implements IStorageProvider { await Promise.allSettled([...dirs].map(async (dir) => rmdir(dir))); } } + + /** + * Resolves a file system path to an absolute path. + * Ensures the path is resolved as an absolute path and properly formatted + * with a leading separator if not already present. + * @param path - The input path string to normalize + * @returns An absolute path with proper path separators + */ + private resolveAbsolutePath(path: string): string { + return resolve(`${path.startsWith(sep) ? '' : sep}${path}`); + } } diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index e392459..f798f12 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -1,3 +1,5 @@ +import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; + /** * A single failed deletion paired with a short reason string (e.g. 'ENOENT', * 'AccessDenied', 'NoSuchKey'). @@ -7,7 +9,13 @@ export interface DeleteFailure { reason: string; } -export interface IStorageProvider { +export interface DeleteResourcesResult { + failures: DeleteFailure[]; +} + +export type StorageProvider = DeleteStoredResourcesParams['storageProvider']; + +export interface IStorageProvider { /** * Deletes a batch of relative file paths within the given storage target. * - S3: storageTarget = bucket name; paths are object keys @@ -18,6 +26,13 @@ export interface IStorageProvider { */ delete: (paths: string[], storageTarget: string) => Promise; + /** + * Deletes ALL objects/files under the given paths. + * - S3: storageTarget = bucket name; paths are root paths to resource(s) + * - FS: storageTarget = base directory; paths are relative paths from mounted dir + */ + deleteResources: (deleteStoredResourcesParams: Extract) => Promise; + /** * Returns true if relativePath exists within storageTarget and contains data. * - S3: storageTarget = bucket, relativePath = key prefix — lists objects (KeyCount > 0) @@ -25,3 +40,7 @@ export interface IStorageProvider { */ targetExists: (storageTarget: string, relativePath: string) => Promise; } + +export type StorageProviders = { + [T in StorageProvider]?: IStorageProvider; +}; diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 54b39c1..5450fa1 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,4 @@ -export type { IStorageProvider, DeleteFailure } from './iStorageProvider'; -export { S3StorageProvider } from './s3StorageProvider'; -export { FsStorageProvider } from './fsStorageProvider'; export { summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; +export { FsStorageProvider } from './fsStorageProvider'; +export type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; +export { S3StorageProvider } from './s3StorageProvider'; diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index d554a20..26d230c 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -1,29 +1,43 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; +import { + DeleteObjectsCommand, + HeadBucketCommand, + ListObjectsV2Command, + NoSuchBucket, + NotFound, + paginateListObjectsV2, + S3Client, + S3ServiceException, + type _Object, +} from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; +import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { ConfigType } from '@common/config'; -import { describeError } from '../errors'; -import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; +import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import { describeError, UnrecoverableError } from '../errors'; +import { getChunk } from '../utils'; const S3_MAX_DELETE_BATCH = 1000; -interface S3Config { +type S3StorageProviderType = Extract; + +export interface S3Config { endpoint: string; accessKeyId: string; secretAccessKey: string; - sslEnabled: boolean; - forcePathStyle: boolean; - region: string; + sslEnabled?: boolean; + forcePathStyle?: boolean; + region?: string; } -export class S3StorageProvider implements IStorageProvider { +export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; public constructor( config: ConfigType, private readonly logger: Logger ) { - const s3Config = config.get('s3') as S3Config; + const s3Config = config.get('storage.s3') as unknown as S3Config; this.s3Client = new S3Client({ endpoint: s3Config.endpoint, credentials: { @@ -36,33 +50,43 @@ export class S3StorageProvider implements IStorageProvider { }); } - public async delete(paths: string[], storageTarget: string): Promise { - if (paths.length === 0) { - return []; + public async delete(paths: string[], bucket: string): Promise { + this.logger.debug({ msg: 'Deleting objects from S3', bucket, count: paths.length }); + + const failures: DeleteFailure[] = []; + + for (const chunk of getChunk(paths, S3_MAX_DELETE_BATCH)) { + const failed = await this.deleteChunk(chunk, bucket); + failures.push(...failed); } - this.logger.debug({ msg: 'Deleting objects from S3', bucket: storageTarget, count: paths.length }); + return failures; + } + public async deleteResources({ + bucket, + paths, + }: Extract): Promise { const failures: DeleteFailure[] = []; + this.logger.debug({ msg: `Starting S3 deletion`, bucket, paths }); - for (const chunk of this.chunk(paths, S3_MAX_DELETE_BATCH)) { - try { - const failed = await this.deleteChunk(chunk, storageTarget); - failures.push(...failed); - } catch (error) { - // Whole chunk failed (network/auth/etc) — mark every path in it with the same reason - // so the caller still gets a per-path failure list and a human-readable cause. - const reason = describeError(error); - this.logger.error({ msg: 'S3 batch request failed', bucket: storageTarget, reason, error }); - failures.push(...chunk.map((path) => ({ path, reason }))); - } + const exists = await this.storageExists(bucket); + if (!exists) { + throw new UnrecoverableError(`Bucket does not exist: ${bucket}`); } - return failures; + if (paths.some((path) => path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion + + for (const path of paths) { + const pathFailures = await this.deleteResource({ bucket, path }); + failures.push(...pathFailures); + } + + return { failures }; } public async targetExists(bucket: string, relativePath: string): Promise { - const prefix = relativePath.endsWith('/') ? relativePath : `${relativePath}/`; + const prefix = this.normalizePrefix(relativePath); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); return (result.KeyCount ?? 0) > 0; @@ -74,20 +98,132 @@ export class S3StorageProvider implements IStorageProvider { } private async deleteChunk(paths: string[], bucket: string): Promise { - const command = new DeleteObjectsCommand({ - Bucket: bucket, - Delete: { Objects: paths.map((Key) => ({ Key })) }, + try { + const command = new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: paths.map((Key) => ({ Key })) }, + }); + + const response = await this.s3Client.send(command); + const errors = (response.Errors ?? []) + .filter((error): error is typeof error & { Key: string } => Boolean(error.Key)) // Only include entries with a Key so every failure maps to a specific path. + .map((error) => ({ path: error.Key, reason: error.Code ?? error.Message ?? 'Unknown' })); + if (errors.length > 0) this.logger.warn({ msg: `Failed to delete ${errors.length} out of ${paths.length} objects` }); + return errors; + } catch (err) { + const reason = describeError(err); + this.logger.error({ msg: 'S3 batch request failed', bucket, reason, error: err }); + return paths.map((path) => { + return { path, reason }; + }); + } + } + + private async deleteResource({ bucket, path }: { path: string; bucket: string }): Promise { + const failures: DeleteFailure[] = []; + const normalizedPrefix = this.normalizePrefix(path); + let totalDeletedObjectsCount = 0, + totalFailedObjectsCount = 0; + + this.logger.debug({ msg: 'Deleting all objects from S3 under given path', bucket, prefix: normalizedPrefix }); + + const s3Objects = this.getS3Objects({ + bucket, + prefix: normalizedPrefix, }); - const response = await this.s3Client.send(command); - return (response.Errors ?? []) - .filter((e): e is typeof e & { Key: string } => Boolean(e.Key)) // Only include entries with a Key so every failure maps to a specific path. - .map((e) => ({ path: e.Key, reason: e.Code ?? e.Message ?? 'Unknown' })); + try { + for await (const pageOfObjects of s3Objects) { + this.logger.debug(`Received a batch of ${pageOfObjects.length} objects to delete`); + const keys = pageOfObjects.map((obj) => obj.Key!).filter(Boolean); + + const chunkFailures = await this.deleteChunk(keys, bucket); + failures.push(...chunkFailures); + + const failedObjectsCount = chunkFailures.length; + const deletedObjectsCount = keys.length - failedObjectsCount; + totalDeletedObjectsCount += deletedObjectsCount; + totalFailedObjectsCount += failedObjectsCount; + + this.logger.debug({ + msg: `Successfully deleted ${deletedObjectsCount} out of total of ${totalDeletedObjectsCount} successfully deleted objects`, + }); + + if (failedObjectsCount > 0) + this.logger.debug({ + msg: `Could not delete ${failedObjectsCount} objects out of total of ${totalFailedObjectsCount} objects that could not be deleted`, + }); + } + this.logger.debug({ msg: 'Deletion completed', prefix: normalizedPrefix, totalDeletedObjectsCount, totalFailedObjectsCount }); + return failures; + } catch (err) { + this.logger.error({ + msg: 'Stream of objects was interrupted by an error', + prefix: normalizedPrefix, + totalDeletedObjectsCount, + totalFailedObjectsCount, + err, + }); + throw err; + } + } + + private async *getS3Objects({ + bucket, + prefix, + pageSize = S3_MAX_DELETE_BATCH, + }: { + bucket: string; + prefix?: string; + pageSize?: number; + }): AsyncGenerator<_Object[], void, unknown> { + const paginatorConfig = { + client: this.s3Client, + pageSize, + }; + + const commandInput = { + Bucket: bucket, + Prefix: prefix, + }; + + try { + const paginator = paginateListObjectsV2(paginatorConfig, commandInput); + + for await (const page of paginator) { + yield page.Contents ?? []; + } + } catch (err) { + if (err instanceof NoSuchBucket) { + this.logger.error({ msg: `S3 Error [${err.name}] no such bucket: ${err.message} (Req ID: ${err.$metadata.requestId})` }); + } else if (err instanceof S3ServiceException) { + this.logger.error({ msg: `S3 Error [${err.name}] occured during pagination: ${err.message} (Req ID: ${err.$metadata.requestId})` }); + } else { + this.logger.error({ msg: 'Unexpected error occurred during pagination', err }); + } + throw err; + } } - private *chunk(paths: string[], size: number): Generator { - for (let i = 0; i < paths.length; i += size) { - yield paths.slice(i, i + size); + private normalizePrefix(path: string): string { + return path.endsWith('/') ? path : `${path}/`; // Ensure prefix ends with '/' so the target is a "folder" contents (e.g. 'photos/' matches 'photos/img.jpg' not 'photos_old/...') + } + + private async storageExists(bucket: string): Promise { + try { + this.logger.debug({ msg: `Checking bucket exists`, bucket }); + const command = new HeadBucketCommand({ Bucket: bucket }); + await this.s3Client.send(command); // If it resolves, the bucket exists and you have permission to access it + this.logger.debug({ msg: `Bucket exists`, bucket }); + return true; + } catch (err) { + if (err instanceof NotFound) { + this.logger.error({ msg: `Bucket does not exist`, bucket, err }); + return false; + } + const reason = describeError(err); + this.logger.error({ msg: 'Failed to check if bucket exists', bucket, reason, error: err }); + throw err; } } } diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts new file mode 100644 index 0000000..a710816 --- /dev/null +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -0,0 +1,66 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { deleteStoredResourcesParamsSchema, type DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; +import type { ConfigType } from '@common/config'; +import { SERVICES } from '@common/constants'; +import { summarizeDeleteFailures, type IStorageProvider, type StorageProvider, type StorageProviders } from '@src/cleaner/storageProviders'; +import { RecoverableError, UnrecoverableError } from '../errors'; +import { validateSchema } from '../utils'; +import type { ITaskStrategy } from './taskStrategy'; + +@injectable() +export class DeleteStoredResourcesStrategy implements ITaskStrategy { + private readonly failureSampleSize: number; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: StorageProviders + ) { + this.failureSampleSize = this.config.get('strategies.storedResourcesDeletion.failureSampleSize') as unknown as number; + } + + public validate(params: unknown): DeleteStoredResourcesParams { + this.logger.debug({ msg: `Validating input parameters` }); + return validateSchema(deleteStoredResourcesParamsSchema, params, this.logger); + } + + public async execute(params: DeleteStoredResourcesParams): Promise { + const { paths } = params; + const provider = this.resolveStorageProvider(params); + + this.logger.info({ msg: 'Starting deletion', provider: params.storageProvider, paths }); + + const { failures } = await provider.deleteResources(params); + + if (failures.length > 0) { + const { counts, summary, sample } = summarizeDeleteFailures(failures, this.failureSampleSize); + this.logger.error({ + msg: 'Deletion failed', + provider: params.storageProvider, + paths, + failureCount: failures.length, + reasonCounts: counts, + sample, + }); + throw new RecoverableError(`Failed to delete resource(s). Reasons: ${summary}. Sample: ${sample.join(', ')}`); + } + + this.logger.info({ + msg: 'Deletion completed successfully', + provider: params.storageProvider, + paths, + }); + } + + private resolveStorageProvider( + params: Extract + ): IStorageProvider { + if (!(params.storageProvider in this.storageProviders)) throw new UnrecoverableError(`Unsupported storage provider ${params.storageProvider}`); + // eslint-disable-next-line @typescript-eslint/naming-convention + const storageProvider = this.storageProviders[params.storageProvider]; + if (storageProvider === undefined) throw new UnrecoverableError(`Unsupported storage provider ${params.storageProvider}`); + this.logger.debug({ msg: `Using ${params.storageProvider} provider` }); + return storageProvider; + } +} diff --git a/src/cleaner/strategies/index.ts b/src/cleaner/strategies/index.ts index f3895d4..2bcd2c4 100644 --- a/src/cleaner/strategies/index.ts +++ b/src/cleaner/strategies/index.ts @@ -1,3 +1,4 @@ -export { type ITaskStrategy } from './taskStrategy'; +export { DeleteStoredResourcesStrategy } from './deleteStoredResourcesStrategy'; export { StrategyFactory, type TaskContext } from './strategyFactory'; +export type { ITaskStrategy } from './taskStrategy'; export { TilesDeletionStrategy } from './tilesDeletionStrategy'; diff --git a/src/cleaner/strategies/strategyFactory.ts b/src/cleaner/strategies/strategyFactory.ts index 00da2be..c56f541 100644 --- a/src/cleaner/strategies/strategyFactory.ts +++ b/src/cleaner/strategies/strategyFactory.ts @@ -1,8 +1,9 @@ -import { container, inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; +import { container, inject, injectable } from 'tsyringe'; import { SERVICES } from '@common/constants'; -import { StrategyNotFoundError } from '../errors'; -import type { ITaskStrategy } from './taskStrategy'; +import { StrategyNotFoundError } from '@src/cleaner/errors'; +import type { ITaskStrategy } from '@src/cleaner/strategies'; +import { getJobAndTaskToken } from '@src/common/dependencyRegistration'; export interface TaskContext { jobId: string; @@ -27,8 +28,10 @@ export class StrategyFactory { public resolveWithContext(taskContext: TaskContext): ITaskStrategy { this.logger.debug({ msg: 'Resolving strategy with task context', ...taskContext }); - if (!container.isRegistered(taskContext.taskType)) { - throw new StrategyNotFoundError(taskContext.taskType); + const jobTaskToken = getJobAndTaskToken(taskContext); + + if (!container.isRegistered(jobTaskToken)) { + throw new StrategyNotFoundError({ jobType: taskContext.jobType, taskType: taskContext.taskType }); } const taskContainer = container.createChildContainer(); @@ -39,7 +42,7 @@ export class StrategyFactory { taskContainer.register(SERVICES.LOGGER, { useValue: taskLogger }); taskContainer.register(SERVICES.TASK_CONTEXT, { useValue: taskContext }); - const strategy = taskContainer.resolve(taskContext.taskType); + const strategy = taskContainer.resolve(jobTaskToken); taskLogger.debug({ msg: 'Strategy resolved successfully with task context' }); diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index c0b1b6c..a151a04 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -1,13 +1,19 @@ -import { inject, injectable } from 'tsyringe'; +import { NoSuchKey } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; -import { SourceType, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { NoSuchKey } from '@aws-sdk/client-s3'; -import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; +import { SourceType, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; import type { ConfigType } from '@common/config'; -import { validateSchema } from '../utils'; +import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; +import { + summarizeDeleteFailures, + type DeleteFailure, + type IStorageProvider, + type StorageProvider, + type StorageProviders, +} from '@src/cleaner/storageProviders'; import { RecoverableError, UnrecoverableError, describeError } from '../errors'; -import { summarizeDeleteFailures, type DeleteFailure, type IStorageProvider } from '../storageProviders'; +import { validateSchema } from '../utils'; import type { TaskContext } from './strategyFactory'; import type { ITaskStrategy } from './taskStrategy'; @@ -24,7 +30,7 @@ export class TilesDeletionStrategy implements ITaskStrategy public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) config: ConfigType, - @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: Map, + @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: StorageProviders, @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient, @inject(SERVICES.TASK_CONTEXT) private readonly taskContext: TaskContext ) { @@ -32,15 +38,16 @@ export class TilesDeletionStrategy implements ITaskStrategy this.concurrency = config.get('strategies.tilesDeletion.concurrency') as unknown as number; this.failureSampleSize = config.get('strategies.tilesDeletion.failureSampleSize') as unknown as number; this.s3Bucket = config.get('strategies.tilesDeletion.s3Bucket') as unknown as string; - this.fsBasePath = config.get('strategies.tilesDeletion.fsBasePath') as unknown as string; + this.fsBasePath = config.get('strategies.tilesDeletion.fsBasePath') as unknown as string; // TODO: merge with fs.basePath } public validate(params: unknown): TilesDeletionParams { + this.logger.debug({ msg: `Validating input parameters` }); return validateSchema(tilesDeletionParamsSchema, params, this.logger); } public async execute(params: TilesDeletionParams): Promise { - const { provider, storageTarget } = this.resolveProvider(params); + const { provider, storageTarget } = this.resolveStorageProvider(params); if (!(await provider.targetExists(storageTarget, params.tilesPath))) { throw new UnrecoverableError(`${params.sourceProvider} storage target does not exist: ${storageTarget}/${params.tilesPath}`); @@ -104,13 +111,16 @@ export class TilesDeletionStrategy implements ITaskStrategy this.logger.info({ msg: 'Tiles deletion completed successfully', deletedCount: totalTiles }); } - private resolveProvider(params: TilesDeletionParams): { provider: IStorageProvider; storageTarget: string } { - const provider = this.storageProviders.get(params.sourceProvider); - if (provider === undefined) { - throw new UnrecoverableError(`Unknown storage provider: ${params.sourceProvider}`); - } + private resolveStorageProvider( + params: Extract + ): { provider: IStorageProvider; storageTarget: string } { + if (!(params.sourceProvider in this.storageProviders)) throw new UnrecoverableError(`Unsupported storage provider ${params.sourceProvider}`); + // eslint-disable-next-line @typescript-eslint/naming-convention + const storageProvider = this.storageProviders[params.sourceProvider]; + if (storageProvider === undefined) throw new UnrecoverableError(`Unsupported storage provider ${params.sourceProvider}`); const storageTarget = params.sourceProvider === SourceType.S3 ? this.s3Bucket : this.fsBasePath; - return { provider, storageTarget }; + this.logger.debug({ msg: `Using ${params.sourceProvider} provider` }); + return { provider: storageProvider, storageTarget }; } private async deleteAllTiles( diff --git a/src/cleaner/utils/chunk.ts b/src/cleaner/utils/chunk.ts new file mode 100644 index 0000000..d419943 --- /dev/null +++ b/src/cleaner/utils/chunk.ts @@ -0,0 +1,5 @@ +export function* getChunk(items: T[], size: number): Generator { + for (let i = 0; i < items.length; i += size) { + yield items.slice(i, i + size); + } +} diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index fc203c9..574dfb3 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,2 +1,3 @@ +export { getChunk } from './chunk'; export { buildPollingPairs } from './pairBuilder'; export { validateSchema } from './validationHelper'; diff --git a/src/common/dependencyRegistration.ts b/src/common/dependencyRegistration.ts index a591ff9..29f34e1 100644 --- a/src/common/dependencyRegistration.ts +++ b/src/common/dependencyRegistration.ts @@ -8,6 +8,8 @@ export interface InjectionObject { provider: Providers; } +export const getJobAndTaskToken = ({ jobType, taskType }: { jobType: string; taskType: string }): string => `${jobType}-${taskType}`; + export const registerDependencies = ( dependencies: InjectionObject[], override?: InjectionObject[], diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 586613b..41ee7fb 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,22 +1,23 @@ -import { getOtelMixin } from '@map-colonies/telemetry'; +import { IWorker, JobnikSDK } from '@map-colonies/jobnik-sdk'; +import { jsLogger, type Logger } from '@map-colonies/js-logger'; +import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { SourceType } from '@map-colonies/raster-shared'; +import { getOtelMixin } from '@map-colonies/telemetry'; import { trace } from '@opentelemetry/api'; import { Registry } from 'prom-client'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; -import { jsLogger, type Logger } from '@map-colonies/js-logger'; -import { IWorker, JobnikSDK } from '@map-colonies/jobnik-sdk'; -import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { SERVICES, SERVICE_NAME } from '@common/constants'; +import { SERVICE_NAME, SERVICES } from '@common/constants'; +import { getJobAndTaskToken, InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { getTracing } from '@common/tracing'; +import type { StorageProviders } from '@src/cleaner/storageProviders'; +import { ErrorHandler } from './cleaner/errors'; +import { JobTrackerClient } from './cleaner/httpClients'; +import { FsStorageProvider, S3StorageProvider } from './cleaner/storageProviders'; +import { DeleteStoredResourcesStrategy, StrategyFactory, TilesDeletionStrategy } from './cleaner/strategies'; import type { QueueConfig } from './cleaner/types'; import { ConfigType, getConfig } from './common/config'; import { workerBuilder } from './worker'; -import { StrategyFactory, TilesDeletionStrategy } from './cleaner/strategies'; -import { ErrorHandler } from './cleaner/errors'; -import { S3StorageProvider, FsStorageProvider, type IStorageProvider } from './cleaner/storageProviders'; -import { JobTrackerClient } from './cleaner/httpClients'; export interface RegisterOptions { override?: InjectionObject[]; @@ -100,22 +101,58 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.STORAGE_PROVIDERS, provider: { - useFactory: instancePerContainerCachingFactory((container) => { + useFactory: instancePerContainerCachingFactory((container) => { const config = container.resolve(SERVICES.CONFIG); const logger = container.resolve(SERVICES.LOGGER); - return new Map([ - [SourceType.S3, new S3StorageProvider(config, logger)], - [SourceType.FS, new FsStorageProvider(logger)], - ]); + const cleanupStorageProviders = config.get('storage.cleanupStorageProviders') as unknown as string[]; + const providers = { + ...(cleanupStorageProviders.includes(SourceType.S3) && { [SourceType.S3]: new S3StorageProvider(config, logger) }), + ...(cleanupStorageProviders.includes(SourceType.FS) && { [SourceType.FS]: new FsStorageProvider(config, logger) }), + }; + return providers; }), }, }, { - token: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, //TODO: when we create worker config schema we can move this to a constant and remove the cast + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.update.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, + }), provider: { useClass: TilesDeletionStrategy, }, }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.swapUpdate.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, + }), + provider: { + useClass: TilesDeletionStrategy, + }, + }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.deleteLayer.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.layerDeletion.type') as unknown as string, + }), + provider: { + useClass: DeleteStoredResourcesStrategy, + }, + }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.deleteLayer.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.artifactsDeletion.type') as unknown as string, + }), + provider: { + useClass: DeleteStoredResourcesStrategy, + }, + }, { token: 'onSignal', provider: { diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 079851f..f100f9c 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -1,12 +1,14 @@ -import { vi } from 'vitest'; import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import type { ConfigType } from '../../src/common/config'; -import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; -import type { IStorageProvider } from '../../src/cleaner/storageProviders'; +import { vi } from 'vitest'; +import type { FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; +import type { IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders/iStorageProvider'; +import type { S3Config } from '@src/cleaner/storageProviders/s3StorageProvider'; import type { ErrorHandler } from '../../src/cleaner/errors'; -import type { ErrorDecision, PollingPairConfig } from '../../src/cleaner/types'; import type { JobTrackerClient } from '../../src/cleaner/httpClients'; +import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; +import type { ErrorDecision, PollingPairConfig } from '../../src/cleaner/types'; +import type { ConfigType } from '../../src/common/config'; import { TaskPoller } from '../../src/worker/taskPoller'; // ─── Logger ────────────────────────────────────────────────────────────────── @@ -66,9 +68,10 @@ export function createMockErrorHandler(defaultDecision: ErrorDecision = { should // ─── StorageProvider ───────────────────────────────────────────────────────── -export function createMockStorageProvider(): IStorageProvider { +export function createMockStorageProvider(): IStorageProvider { return { delete: vi.fn().mockResolvedValue([]), + deleteResources: vi.fn().mockResolvedValue({ failures: [] }), targetExists: vi.fn().mockResolvedValue(true), }; } @@ -95,6 +98,20 @@ export function createMockStrategyConfig(overrides: Record = {} return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; } +// ─── Strategy Config (DeleteStoredResourcesStrategy) ──────────────────────────── + +export const STORED_RESOURCES_DELETION_CONFIG_DEFAULTS = { + failureSampleSize: 3, +} as const; + +export function createMockStoredResourcesDeletionStrategyConfig(overrides: Record = {}): ConfigType { + const values: Record = { + 'strategies.storedResourcesDeletion.failureSampleSize': STORED_RESOURCES_DELETION_CONFIG_DEFAULTS.failureSampleSize, + ...overrides, + }; + return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; +} + // ─── S3 Storage Config (S3StorageProvider) ─────────────────────────────────── export const S3_STORAGE_CONFIG_DEFAULTS = { @@ -104,7 +121,7 @@ export const S3_STORAGE_CONFIG_DEFAULTS = { sslEnabled: false, forcePathStyle: true, region: 'us-east-1', -} as const; +} as const satisfies S3Config; export function createMockS3Config(): ConfigType { return { @@ -112,6 +129,18 @@ export function createMockS3Config(): ConfigType { } as unknown as ConfigType; } +// ─── FS Storage Config (FsStorageProvider) ─────────────────────────────────── + +export const FS_STORAGE_CONFIG_DEFAULTS = { + basePath: '/test/tiles', +} as const satisfies FsConfig; + +export function createMockFsConfig(): ConfigType { + return { + get: vi.fn().mockReturnValue({ ...FS_STORAGE_CONFIG_DEFAULTS }), + } as unknown as ConfigType; +} + // ─── JobTrackerClient ───────────────────────────────────────────────────────── export function createMockJobTrackerClient(): JobTrackerClient { diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index cdca3f4..6138488 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -1,30 +1,49 @@ -import { stat, unlink, rmdir } from 'node:fs/promises'; +import { accessSync, Stats, statSync } from 'node:fs'; +import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; -import { Stats } from 'node:fs'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { FsStorageProvider } from '@src/cleaner/storageProviders/fsStorageProvider'; -import { createMockLogger } from '../helpers/mocks'; +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; +import { FsStorageProvider, type FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; +import type { ConfigType } from '@src/common/config'; +import { createMockFsConfig, createMockLogger, FS_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; vi.mock('node:fs/promises', () => ({ stat: vi.fn(), unlink: vi.fn(), rmdir: vi.fn(), + rm: vi.fn(), })); -const BASE_PATH = '/tiles/test'; +vi.mock(import('node:fs'), async (importOriginal) => { + const originModule = await importOriginal(); + return { + ...originModule, + accessSync: vi.fn(), + statSync: vi.fn(), + }; +}); +const BASE_PATH = FS_STORAGE_CONFIG_DEFAULTS.basePath; describe('FsStorageProvider', () => { let provider: FsStorageProvider; + let mockLogger: Logger; + let mockConfig: ConfigType; beforeEach(() => { vi.clearAllMocks(); vi.mocked(stat).mockResolvedValue({} as Stats); vi.mocked(unlink).mockResolvedValue(undefined); vi.mocked(rmdir).mockResolvedValue(undefined); - provider = new FsStorageProvider(createMockLogger()); + vi.mocked(rm).mockResolvedValue(undefined); + vi.mocked(accessSync).mockReturnValue(undefined); + vi.mocked(statSync).mockReturnValue({ isDirectory: () => true } as Stats); + mockLogger = createMockLogger(); + mockConfig = createMockFsConfig(); + provider = new FsStorageProvider(mockConfig, mockLogger); }); - describe('targetExists', () => { + describe('#targetExists', () => { const RELATIVE_PATH = 'layer/v1'; it('should call stat with full target path', async () => { @@ -50,7 +69,7 @@ describe('FsStorageProvider', () => { }); }); - describe('delete', () => { + describe('#delete', () => { it('should return empty array for empty input', async () => { const result = await provider.delete([], BASE_PATH); expect(result).toEqual([]); @@ -182,4 +201,108 @@ describe('FsStorageProvider', () => { }); }); }); + + describe('#deleteResources', () => { + const RELATIVE_PATH = 'layer/v1'; + + it('should successfully call delete all files and return without failures', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, RELATIVE_PATH), { recursive: true, force: true }); + expect(result).toEqual({ failures: [] }); + }); + + it('should throw UnrecoverableError when a path escapes the base path via traversal', async () => { + await expect(provider.deleteResources({ paths: ['../../../etc/passwd'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when only one of several paths escapes the base path', async () => { + await expect(provider.deleteResources({ paths: [RELATIVE_PATH, '../../escape'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path resolves to the base path itself', async () => { + await expect(provider.deleteResources({ paths: [''], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path resolves to the base path itself via "."', async () => { + await expect(provider.deleteResources({ paths: ['.'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should return failures entry when rm rejects', async () => { + vi.mocked(rm).mockRejectedValue(new Error('EACCES')); + + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + + expect(result).toEqual({ + failures: [{ path: join(BASE_PATH, RELATIVE_PATH), reason: 'EACCES' }], + }); + }); + + it('should not throw when rm rejects', async () => { + vi.mocked(rm).mockRejectedValue(new Error('Permission denied')); + + const result = provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + + await expect(result).resolves.not.toThrow(); + }); + }); + + describe('#constructor', () => { + it('should construct successfully when the base path is accessible and is a directory', () => { + expect(() => new FsStorageProvider(mockConfig, mockLogger)).not.toThrow(); + expect(accessSync).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath, expect.any(Number)); + expect(statSync).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath); + }); + + it('should throw ConfigurationError when the base path does not exist', () => { + vi.mocked(accessSync).mockImplementationOnce(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + + expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when access to the base path is denied (EACCES)', () => { + vi.mocked(accessSync).mockImplementationOnce(() => { + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); + }); + + expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when access to the base path is denied (EPERM)', () => { + vi.mocked(accessSync).mockImplementationOnce(() => { + throw Object.assign(new Error('EPERM'), { code: 'EPERM' }); + }); + + expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when the base path exists but is not a directory', () => { + vi.mocked(statSync).mockReturnValueOnce({ isDirectory: () => false } as Stats); + + expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + }); + + it('should resolve a relative base path (without a leading separator) to an absolute path', () => { + const relativeConfig = { + get: vi.fn().mockReturnValue({ basePath: 'relative/tiles' } satisfies FsConfig), + } as unknown as ConfigType; + + expect(() => new FsStorageProvider(relativeConfig, createMockLogger())).not.toThrow(); + expect(accessSync).toHaveBeenCalledWith('/relative/tiles', expect.any(Number)); + }); + + it('should throw ConfigurationError for an unexpected accessibility error', () => { + vi.mocked(accessSync).mockImplementationOnce(() => { + throw new Error('disk exploded'); + }); + + expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + }); + }); }); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 1dde516..008cc35 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -1,23 +1,40 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; +import { + DeleteObjectsCommand, + ListObjectsV2Command, + NoSuchBucket, + NotFound, + paginateListObjectsV2, + S3Client, + S3ServiceException, +} from '@aws-sdk/client-s3'; +import { faker } from '@faker-js/faker'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { UnrecoverableError } from '@src/cleaner/errors'; import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; const mockSend = vi.fn(); +const mockPaginateListObjectsV2Next = vi.fn(); +const mockPaginateListObjectsV2Return = vi.fn(); +const mockPaginateListObjectsV2Throw = vi.fn(); +const mockPaginateListObjectsV2Iterator = vi.fn().mockReturnValue({ + next: mockPaginateListObjectsV2Next, + return: mockPaginateListObjectsV2Return, + throw: mockPaginateListObjectsV2Throw, +}); +const mockPaginateListObjectsV2 = { + [Symbol.asyncIterator]: mockPaginateListObjectsV2Iterator, +}; -vi.mock('@aws-sdk/client-s3', () => { - class NoSuchBucket extends Error { - public constructor() { - super('NoSuchBucket'); - this.name = 'NoSuchBucket'; - } - } +vi.mock(import('@aws-sdk/client-s3'), async (importOriginal) => { + const originModule = await importOriginal(); return { - S3Client: vi.fn(() => ({ send: mockSend })), - DeleteObjectsCommand: vi.fn((input: unknown) => input), - ListObjectsV2Command: vi.fn((input: unknown) => input), - NoSuchBucket, + ...originModule, + S3Client: vi.fn(() => ({ send: mockSend })) as unknown as typeof S3Client, + DeleteObjectsCommand: vi.fn((input: unknown) => input) as unknown as typeof DeleteObjectsCommand, + ListObjectsV2Command: vi.fn((input: unknown) => input) as unknown as typeof ListObjectsV2Command, + paginateListObjectsV2: vi.fn(() => mockPaginateListObjectsV2) as unknown as typeof paginateListObjectsV2, }; }); @@ -28,11 +45,14 @@ describe('S3StorageProvider', () => { beforeEach(() => { vi.clearAllMocks(); - mockSend.mockResolvedValue({ Errors: [] }); provider = new S3StorageProvider(createMockS3Config(), createMockLogger()); }); - describe('delete', () => { + describe('#delete', () => { + beforeEach(() => { + mockSend.mockResolvedValue({ Errors: [] }); + }); + it('should return empty array for empty input', async () => { const result = await provider.delete([], BUCKET); expect(result).toEqual([]); @@ -164,7 +184,11 @@ describe('S3StorageProvider', () => { }); }); - describe('targetExists', () => { + describe('#targetExists', () => { + beforeEach(() => { + mockSend.mockResolvedValue({ Errors: [] }); + }); + const PREFIX = 'some/prefix'; it('should list objects with bucket and relativePath prefix', async () => { @@ -215,7 +239,281 @@ describe('S3StorageProvider', () => { }); }); - describe('constructor', () => { + describe('#deleteResources', () => { + const PATH = 'layer/v1'; + const NORMALIZED_PATH = `${PATH}/`; + + it('should throw UnrecoverableError when bucket does not exist', async () => { + mockSend.mockRejectedValueOnce(new NotFound({ $metadata: {}, message: '' })); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + }); + + it('should throw UnrecoverableError when storage check failing', async () => { + const expectedError = new Error('error'); + mockSend.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + }); + + it('should throw UnrecoverableError when a path is empty (root deletion guard)', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + + const result = provider.deleteResources({ paths: [''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(paginateListObjectsV2).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when any path is empty even if others are valid', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + + const result = provider.deleteResources({ paths: [PATH, ''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + }); + + it('should return empty result when listing returns no keys', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should normalize prefix by appending trailing slash', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ Prefix: NORMALIZED_PATH })); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should not double-add trailing slash when prefix already ends with one', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + await provider.deleteResources({ paths: [`${PATH}/`], bucket: BUCKET, storageProvider: 'S3' }); + + expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ Prefix: NORMALIZED_PATH })); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should list and delete a single page of objects', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should list and delete multiple pages of objects', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({ Errors: [] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete multiple paths and their objects', async () => { + const PATH1 = 'layer/v1'; + const PATH2 = 'layer/v2'; + const path1Page1Keys = ['layer/v1/0/0.png']; + const path1Page2Keys = ['layer/v1/0/1.png']; + const path2Page1Keys = ['layer/v2/0/0.png']; + const path2Page2Keys = ['layer/v2/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: path1Page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: path1Page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }) + .mockResolvedValueOnce({ done: false, value: { Contents: path2Page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: path2Page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 1 + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 2 + .mockResolvedValueOnce({ Errors: [] }) // delete path 2 page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete path 2 page 2 + + const result = await provider.deleteResources({ paths: [PATH1, PATH2], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(6); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(5); + }); + + it('should handle empty page Contents response', async () => { + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: false, value: {} }).mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should handle empty delete page Errors response', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({}); // delete page 1 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should paginate and return all delete errors returned by S3 per object', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [{ path: 'layer/v1/0/0.png', reason: 'AccessDenied' }] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should paginate and return all delete errors thrown and unhandled by S3', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new Error('NetworkError')) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [{ path: 'layer/v1/0/0.png', reason: 'NetworkError' }] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should stop pagination and throw an error when bucket does not exist', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + const expectedError = new NoSuchBucket({ $metadata: { requestId: faker.string.uuid() }, message: 'msg' }); + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: '', storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should stop pagination and throw an error when S3 throws a service error', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + const expectedError = new S3ServiceException({ $fault: 'server', $metadata: { requestId: faker.string.uuid() }, name: 'msg' }); + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should stop pagination and throw an error when S3 throws a bucket does not exists error', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + const expectedError = new NoSuchBucket({ $metadata: {}, message: '' }); + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + + it('should stop pagination and return failure when list call throws', async () => { + mockSend.mockResolvedValueOnce(undefined); // bucket exists + const expectedError = new Error('NetworkError'); + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + }); + }); + + describe('#constructor', () => { it('should construct S3Client with config values', () => { expect(S3Client).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/tests/strategies/deleteStoredResourcesStrategy.spec.ts b/tests/strategies/deleteStoredResourcesStrategy.spec.ts new file mode 100644 index 0000000..302b194 --- /dev/null +++ b/tests/strategies/deleteStoredResourcesStrategy.spec.ts @@ -0,0 +1,135 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { SourceType, type DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { beforeEach, describe, expect, it, type vi } from 'vitest'; +import { RecoverableError, UnrecoverableError, ValidationError } from '@src/cleaner/errors'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; +import { DeleteStoredResourcesStrategy } from '@src/cleaner/strategies/deleteStoredResourcesStrategy'; +import type { ConfigType } from '@src/common/config'; +import { createMockStoredResourcesDeletionStrategyConfig, createMockLogger, createMockStorageProvider } from '../helpers/mocks'; + +const S3_BUCKET = 'test-bucket'; +const s3Params = { storageProvider: SourceType.S3, paths: ['layer1'], bucket: S3_BUCKET }; +const fsParams = { storageProvider: SourceType.FS, paths: ['layer2'] }; + +describe('DeleteStoredResourcesStrategy', () => { + let strategy: DeleteStoredResourcesStrategy; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockS3Provider: IStorageProvider<'S3'>; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockFsProvider: IStorageProvider<'FS'>; + let mockLogger: Logger; + let mockConfig: ConfigType; + + beforeEach(() => { + mockS3Provider = createMockStorageProvider(); + mockFsProvider = createMockStorageProvider(); + mockLogger = createMockLogger(); + mockConfig = createMockStoredResourcesDeletionStrategyConfig(); + + const storageProviders: StorageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: mockS3Provider, + }; + + strategy = new DeleteStoredResourcesStrategy(mockLogger, mockConfig, storageProviders); + }); + + describe('#validate', () => { + it('should validate and return S3 params', () => { + const result = strategy.validate(s3Params); + + expect(result).toEqual(s3Params); + }); + + it('should validate and return FS params', () => { + const result = strategy.validate(fsParams); + + expect(result).toEqual(fsParams); + }); + + it('should throw ValidationError when storageProvider is missing', () => { + expect(() => strategy.validate({ catalogId: 'layer1' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when storageProvider is unknown', () => { + expect(() => strategy.validate({ storageProvider: 'GCS', catalogId: 'layer1' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when tilesPath is empty string', () => { + expect(() => strategy.validate({ storageProvider: SourceType.S3, catalogId: '' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when tilesPath is missing', () => { + expect(() => strategy.validate({ storageProvider: SourceType.S3 })).toThrow(ValidationError); + }); + + it('should throw ValidationError for null params', () => { + expect(() => strategy.validate(null)).toThrow(ValidationError); + }); + }); + + describe('#execute', () => { + it('should call delete all resources on S3 provider', async () => { + await strategy.execute(s3Params); + + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: s3Params.paths, bucket: S3_BUCKET, storageProvider: 'S3' }); + expect(mockFsProvider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should call delete all resources on FS provider', async () => { + await strategy.execute(fsParams); + + expect(mockFsProvider.deleteResources).toHaveBeenCalledWith({ paths: fsParams.paths, storageProvider: 'FS' }); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should resolve without throwing when deleteResources returns no failures', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ failures: [] }); + + const result = strategy.execute(s3Params); + + await expect(result).resolves.toBeUndefined(); + }); + + it('should throw UnrecoverableError for unknown provider', async () => { + const unknownParams = { ...s3Params, storageProvider: 'UNKNOWN' } as unknown as DeleteStoredResourcesParams; + + const result = strategy.execute(unknownParams); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when the provider is registered but resolves to undefined', async () => { + const storageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: undefined, + } as unknown as StorageProviders; + strategy = new DeleteStoredResourcesStrategy(mockLogger, mockConfig, storageProviders); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should throw RecoverableError when deleteResources returns failures', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ + failures: [{ path: 'layer1/0/0.png', reason: 'AccessDenied' }], + }); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(RecoverableError); + }); + + it('should rethrow error thrown by deleteResources', async () => { + const expectedError = new Error('Custom'); + (mockS3Provider.deleteResources as ReturnType).mockRejectedValueOnce(expectedError); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(expectedError); + }); + }); +}); diff --git a/tests/strategyFactory.spec.ts b/tests/strategyFactory.spec.ts index 2daa995..027a1f1 100644 --- a/tests/strategyFactory.spec.ts +++ b/tests/strategyFactory.spec.ts @@ -1,11 +1,13 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { container } from 'tsyringe'; import { faker } from '@faker-js/faker'; import type { Logger } from '@map-colonies/js-logger'; -import { SERVICES } from '../src/common/constants'; -import { StrategyFactory, TilesDeletionStrategy, type ITaskStrategy, type TaskContext } from '../src/cleaner/strategies'; +import { SourceType } from '@map-colonies/raster-shared'; +import { container } from 'tsyringe'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; import { StrategyNotFoundError } from '../src/cleaner/errors'; -import { createMockLogger, createMockConfig, createMockQueueClient } from './helpers/mocks'; +import { StrategyFactory, TilesDeletionStrategy, type ITaskStrategy, type TaskContext } from '../src/cleaner/strategies'; +import { SERVICES } from '../src/common/constants'; +import { createMockConfig, createMockLogger, createMockQueueClient, createMockStorageProvider } from './helpers/mocks'; class MockStrategy implements ITaskStrategy { public validate(params: unknown): Record { @@ -20,13 +22,25 @@ class MockStrategy implements ITaskStrategy { describe('StrategyFactory', () => { let strategyFactory: StrategyFactory; let mockLogger: Logger; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockS3Provider: IStorageProvider<'S3'>; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockFsProvider: IStorageProvider<'FS'>; beforeEach(() => { mockLogger = createMockLogger(); + mockS3Provider = createMockStorageProvider(); + mockFsProvider = createMockStorageProvider(); + + const storageProviders: StorageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: mockS3Provider, + }; + container.register(SERVICES.LOGGER, { useValue: mockLogger }); container.register(SERVICES.CONFIG, { useValue: createMockConfig() }); - container.register(SERVICES.STORAGE_PROVIDERS, { useValue: new Map() }); + container.register(SERVICES.STORAGE_PROVIDERS, { useValue: storageProviders }); container.register(SERVICES.QUEUE_CLIENT, { useValue: createMockQueueClient() }); strategyFactory = new StrategyFactory(mockLogger); @@ -37,15 +51,16 @@ describe('StrategyFactory', () => { container.clearInstances(); }); - describe('resolveWithContext', () => { + describe('#resolveWithContext', () => { it('should resolve registered strategy with enriched logger context', () => { + const jobType = 'Ingestion_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'Ingestion_Update', + jobType, taskType, }; @@ -63,13 +78,14 @@ describe('StrategyFactory', () => { }); it('should create child logger with task context', () => { + const jobType = 'Ingestion_Swap_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'Ingestion_Swap_Update', + jobType, taskType, }; @@ -96,11 +112,12 @@ describe('StrategyFactory', () => { }); it('should create separate instances for different tasks (child container isolation)', () => { + const jobType = 'Ingestion_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); - const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType }; - const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType }; + const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType }; + const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType }; const strategy1 = strategyFactory.resolveWithContext(context1); const strategy2 = strategyFactory.resolveWithContext(context2); @@ -109,14 +126,15 @@ describe('StrategyFactory', () => { }); it('should resolve different strategies for different task types', () => { + const jobType = 'Ingestion_Update'; const taskType1 = 'tiles-deletion'; const taskType2 = 'files-deletion'; - container.register(taskType1, { useClass: TilesDeletionStrategy }); - container.register(taskType2, { useClass: MockStrategy }); + container.register(`${jobType}-${taskType1}`, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType2}`, { useClass: MockStrategy }); - const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType: taskType1 }; - const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType: taskType2 }; + const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType: taskType1 }; + const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType: taskType2 }; const strategy1 = strategyFactory.resolveWithContext(context1); const strategy2 = strategyFactory.resolveWithContext(context2); @@ -127,13 +145,14 @@ describe('StrategyFactory', () => { }); it('should handle special characters in task type', () => { + const jobType = 'CustomJob'; const taskType = 'task-with-special_chars.v2'; - container.register(taskType, { useClass: MockStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: MockStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'CustomJob', + jobType, taskType, }; diff --git a/tests/tilesDeletionStrategy.spec.ts b/tests/tilesDeletionStrategy.spec.ts index 7669a80..150a3d9 100644 --- a/tests/tilesDeletionStrategy.spec.ts +++ b/tests/tilesDeletionStrategy.spec.ts @@ -1,11 +1,11 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; -import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { faker } from '@faker-js/faker'; -import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; +import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { RecoverableError, UnrecoverableError, ValidationError } from '@src/cleaner/errors'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; import type { TaskContext } from '@src/cleaner/strategies/strategyFactory'; -import { ValidationError, RecoverableError, UnrecoverableError } from '@src/cleaner/errors'; -import type { IStorageProvider } from '@src/cleaner/storageProviders'; +import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from './helpers/mocks'; const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH } = TILES_DELETION_CONFIG_DEFAULTS; @@ -28,8 +28,8 @@ const tilePath = (z: number, x: number, y: number): string => `${s3Params.tilesP describe('TilesDeletionStrategy', () => { let strategy: TilesDeletionStrategy; - let MockS3Provider: IStorageProvider; - let MockFsProvider: IStorageProvider; + let MockS3Provider: IStorageProvider<'S3'>; + let MockFsProvider: IStorageProvider<'FS'>; let mockUpdateProgress: ReturnType; beforeEach(() => { @@ -37,10 +37,10 @@ describe('TilesDeletionStrategy', () => { MockFsProvider = createMockStorageProvider(); mockUpdateProgress = vi.fn().mockResolvedValue(undefined); - const storageProviders = new Map([ - [SourceType.S3, MockS3Provider], - [SourceType.FS, MockFsProvider], - ]); + const storageProviders: StorageProviders = { + [SourceType.FS]: MockFsProvider, + [SourceType.S3]: MockS3Provider, + }; const queueClient = { updateProgress: mockUpdateProgress } as unknown as QueueClient; strategy = new TilesDeletionStrategy(createMockLogger(), createMockStrategyConfig(), storageProviders, queueClient, TASK_CONTEXT); @@ -138,6 +138,24 @@ describe('TilesDeletionStrategy', () => { await expect(strategy.execute(unknownParams)).rejects.toThrow(UnrecoverableError); }); + + it('should throw UnrecoverableError when the provider is registered but resolves to undefined', async () => { + const storageProviders = { + [SourceType.FS]: MockFsProvider, + [SourceType.S3]: undefined, + } satisfies StorageProviders; + strategy = new TilesDeletionStrategy( + createMockLogger(), + createMockStrategyConfig(), + storageProviders, + { updateProgress: mockUpdateProgress } as unknown as QueueClient, + TASK_CONTEXT + ); + + await expect(strategy.execute(s3Params)).rejects.toThrow(UnrecoverableError); + expect(MockS3Provider.targetExists).not.toHaveBeenCalled(); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); + }); }); describe('tile path generation', () => { From d794f6e9b9a7b3a3b45ef7cae7bc01d155cfa7fd Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:16:29 +0300 Subject: [PATCH 02/66] test: move test file to strategies dir --- tests/{ => strategies}/tilesDeletionStrategy.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/{ => strategies}/tilesDeletionStrategy.spec.ts (99%) diff --git a/tests/tilesDeletionStrategy.spec.ts b/tests/strategies/tilesDeletionStrategy.spec.ts similarity index 99% rename from tests/tilesDeletionStrategy.spec.ts rename to tests/strategies/tilesDeletionStrategy.spec.ts index 150a3d9..161a2c7 100644 --- a/tests/tilesDeletionStrategy.spec.ts +++ b/tests/strategies/tilesDeletionStrategy.spec.ts @@ -6,7 +6,7 @@ import { RecoverableError, UnrecoverableError, ValidationError } from '@src/clea import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; import type { TaskContext } from '@src/cleaner/strategies/strategyFactory'; import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; -import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from './helpers/mocks'; +import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from '../helpers/mocks'; const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH } = TILES_DELETION_CONFIG_DEFAULTS; From 31fc94753a106bc9ffec8a6abef0a23132370b98 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:38:03 +0300 Subject: [PATCH 03/66] feat: add delete batch size configuration for s3 and fs storage providers --- config/custom-environment-variables.json | 6 +++++ config/default.json | 6 +++++ helm/templates/configmap.yaml | 5 +++- helm/values.yaml | 4 +++ .../storageProviders/fsStorageProvider.ts | 14 ++++++----- .../storageProviders/s3StorageProvider.ts | 25 +++++++++++-------- tests/helpers/mocks.ts | 14 ++++++++--- .../s3StorageProvider.spec.ts | 15 +++++++++-- 8 files changed, 65 insertions(+), 24 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 47787f9..98c242d 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -79,6 +79,9 @@ "__format": "json" }, "s3": { + "delete": { + "batchSize": "S3_DELETE_BATCH_SIZE" + }, "endpoint": "S3_ENDPOINT", "accessKeyId": "S3_ACCESS_KEY_ID", "secretAccessKey": "S3_SECRET_ACCESS_KEY", @@ -93,6 +96,9 @@ "region": "S3_REGION" }, "fs": { + "delete": { + "batchSize": "FS_DELETE_BATCH_SIZE" + }, "basePath": "FS_BASE_PATH" } }, diff --git a/config/default.json b/config/default.json index d2be632..999d024 100644 --- a/config/default.json +++ b/config/default.json @@ -83,6 +83,9 @@ "storage": { "cleanupStorageProviders": ["FS", "S3"], "s3": { + "delete": { + "batchSize": 1000 + }, "endpoint": "http://localhost:9000", "accessKeyId": "minioadmin", "secretAccessKey": "minioadmin", @@ -91,6 +94,9 @@ "region": "us-east-1" }, "fs": { + "delete": { + "batchSize": 1000 + }, "basePath": "/tiles" } }, diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 8051b0f..a5700d7 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -2,7 +2,8 @@ {{- $serviceUrls := fromYaml (include "common.serviceUrls.merged" .) -}} {{- $storage := fromYaml (include "common.storage.merged" .) -}} {{- $s3 := ($storage.s3) | default dict -}} -{{- $internalPvc := (($storage.fs).internalPvc) | default dict -}} +{{- $fs := ($storage.fs) | default dict -}} +{{- $internalPvc := (($fs).internalPvc) | default dict -}} {{- $tilesFSBasePath := clean (printf "/%s/%s" $internalPvc.mountPath ($internalPvc.tilesSubPath | default "tiles")) -}} {{- if .Values.enabled -}} apiVersion: v1 @@ -55,12 +56,14 @@ data: JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }} {{- end }} {{- if has "S3" $storage.cleanupStorageProviders }} + S3_DELETE_BATCH_SIZE: {{ $s3.delete.batchSize | default 1000 | quote }} S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }} S3_FORCE_PATH_STYLE: {{ $s3.forcePathStyle | default false | quote }} S3_SSL_ENABLED: {{ $s3.sslEnabled | default false | quote }} S3_REGION: {{ $s3.region | default "us-east-1" | quote }} {{- end }} {{- if has "FS" $storage.cleanupStorageProviders }} + FS_DELETE_BATCH_SIZE: {{ $fs.delete.batchSize | default 1000 | quote }} FS_BASE_PATH: {{ $tilesFSBasePath | quote }} {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 3eca562..2203dec 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -17,6 +17,8 @@ storage: - FS - S3 s3: + delete: + batchSize: 1000 endpointUrl: "" forcePathStyle: true secretName: "" @@ -24,6 +26,8 @@ storage: region: "" tilesBucket: "" fs: + delete: + batchSize: 1000 internalPvc: enabled: false name: "" diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index b8f1a35..0474705 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -8,23 +8,25 @@ import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; import { getChunk } from '../utils'; -const FS_MAX_DELETE_BATCH = 1000; - type FSStorageProviderType = Extract; export interface FsConfig { + delete: { + batchSize: number; + }; basePath: string; } export class FsStorageProvider implements IStorageProvider<'FS'> { + private readonly fsConfig: FsConfig; private readonly basePath: string; public constructor( private readonly config: ConfigType, private readonly logger: Logger ) { - const fsConfig = this.config.get('storage.fs') as unknown as FsConfig; - this.basePath = this.resolveAbsolutePath(fsConfig.basePath); + this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; + this.basePath = this.resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); this.logger.debug(`Using ${this.basePath} as base path for FS`); } @@ -43,7 +45,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { this.logger.info({ msg: 'Deleting files from filesystem', basePath: storageTarget, count: paths.length }); const failures: DeleteFailure[] = []; - for (const relativePaths of getChunk(paths, FS_MAX_DELETE_BATCH)) { + for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( relativePaths.map(async (relativePath) => { await unlink(join(storageTarget, relativePath)); @@ -77,7 +79,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { throw new UnrecoverableError(`Cannot delete base path itself`); const failures: DeleteFailure[] = []; - for (const relativePaths of getChunk(paths, FS_MAX_DELETE_BATCH)) { + for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( relativePaths.map(async (relativePath) => { await rm(join(this.basePath, relativePath), { recursive: true, force: true }); diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 26d230c..4865207 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -17,11 +17,12 @@ import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StoragePro import { describeError, UnrecoverableError } from '../errors'; import { getChunk } from '../utils'; -const S3_MAX_DELETE_BATCH = 1000; - type S3StorageProviderType = Extract; export interface S3Config { + delete: { + batchSize: number; + }; endpoint: string; accessKeyId: string; secretAccessKey: string; @@ -32,21 +33,22 @@ export interface S3Config { export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; + private readonly s3Config: S3Config; public constructor( config: ConfigType, private readonly logger: Logger ) { - const s3Config = config.get('storage.s3') as unknown as S3Config; + this.s3Config = config.get('storage.s3') as unknown as S3Config; this.s3Client = new S3Client({ - endpoint: s3Config.endpoint, + endpoint: this.s3Config.endpoint, credentials: { - accessKeyId: s3Config.accessKeyId, - secretAccessKey: s3Config.secretAccessKey, + accessKeyId: this.s3Config.accessKeyId, + secretAccessKey: this.s3Config.secretAccessKey, }, - forcePathStyle: s3Config.forcePathStyle, - region: s3Config.region, - tls: s3Config.sslEnabled, + forcePathStyle: this.s3Config.forcePathStyle, + region: this.s3Config.region, + tls: this.s3Config.sslEnabled, }); } @@ -55,7 +57,7 @@ export class S3StorageProvider implements IStorageProvider = {}): ConfigType { return { - get: vi.fn().mockReturnValue({ ...S3_STORAGE_CONFIG_DEFAULTS }), + get: vi.fn().mockReturnValue({ ...S3_STORAGE_CONFIG_DEFAULTS, ...overrides }), } as unknown as ConfigType; } // ─── FS Storage Config (FsStorageProvider) ─────────────────────────────────── export const FS_STORAGE_CONFIG_DEFAULTS = { + delete: { + batchSize: 3, + }, basePath: '/test/tiles', } as const satisfies FsConfig; -export function createMockFsConfig(): ConfigType { +export function createMockFsConfig(overrides: Record = {}): ConfigType { return { - get: vi.fn().mockReturnValue({ ...FS_STORAGE_CONFIG_DEFAULTS }), + get: vi.fn().mockReturnValue({ ...FS_STORAGE_CONFIG_DEFAULTS, ...overrides }), } as unknown as ConfigType; } diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 008cc35..4b5e1f0 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -9,9 +9,11 @@ import { S3ServiceException, } from '@aws-sdk/client-s3'; import { faker } from '@faker-js/faker'; +import type { Logger } from '@map-colonies/js-logger'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { UnrecoverableError } from '@src/cleaner/errors'; import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; +import type { ConfigType } from '@src/common/config'; import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; const mockSend = vi.fn(); @@ -42,10 +44,14 @@ const BUCKET = 'test-bucket'; describe('S3StorageProvider', () => { let provider: S3StorageProvider; + let mockConfig: ConfigType; + let mockLogger: Logger; beforeEach(() => { vi.clearAllMocks(); - provider = new S3StorageProvider(createMockS3Config(), createMockLogger()); + mockConfig = createMockS3Config(); + mockLogger = createMockLogger(); + provider = new S3StorageProvider(mockConfig, mockLogger); }); describe('#delete', () => { @@ -143,7 +149,12 @@ describe('S3StorageProvider', () => { it('should batch paths into chunks of 1000 (S3 limit)', async () => { const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); - + const mockConfig = createMockS3Config({ + delete: { + batchSize: 1000, + }, + }); + provider = new S3StorageProvider(mockConfig, mockLogger); await provider.delete(paths, BUCKET); expect(mockSend).toHaveBeenCalledTimes(2); From f98067305becf9a8d4ec5498d7c0117facd47f1f Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:41:12 +0300 Subject: [PATCH 04/66] chore: remove extra space in start:dev script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b6e4ebb..d15fb70 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "prebuild": "npm run clean", "build": "tsc --project tsconfig.build.json && tsc-alias -p tsconfig.build.json && npm run assets:copy", "start": "npm run build && cd dist && node --import ./instrumentation.mjs ./index.js", - "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./index.js", + "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./index.js", "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles ./package.json dist", "clean": "rimraf dist", "prepare": "node .husky/install.mjs" From 03538001e9681c8a61cc9a699627f52783767f56 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:57:40 +0300 Subject: [PATCH 05/66] refactor: remove and update logs --- src/cleaner/storageProviders/fsStorageProvider.ts | 2 -- src/cleaner/storageProviders/s3StorageProvider.ts | 2 -- src/cleaner/strategies/deleteStoredResourcesStrategy.ts | 8 +++++++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 0474705..e6f7a48 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -71,8 +71,6 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, }: Extract): Promise { - this.logger.info({ msg: 'Deleting multiple directories/files from filesystem', basePath: this.basePath, count: paths.length }); - // prevent path traversal (i.e. accessing folders above root folder) if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); if (!paths.every((path) => this.resolveAbsolutePath(join(this.basePath, path)) !== this.basePath)) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 4865207..3f106cc 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -127,8 +127,6 @@ export class S3StorageProvider implements IStorageProvider Date: Wed, 15 Jul 2026 23:40:22 +0300 Subject: [PATCH 06/66] test: fix test mocked config returned value --- tests/storageProviders/fsStorageProvider.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index 6138488..568d4e5 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -290,7 +290,7 @@ describe('FsStorageProvider', () => { it('should resolve a relative base path (without a leading separator) to an absolute path', () => { const relativeConfig = { - get: vi.fn().mockReturnValue({ basePath: 'relative/tiles' } satisfies FsConfig), + get: vi.fn().mockReturnValue({ basePath: 'relative/tiles', delete: { batchSize: 100 } } satisfies FsConfig), } as unknown as ConfigType; expect(() => new FsStorageProvider(relativeConfig, createMockLogger())).not.toThrow(); From e852150cd4f49422a4a624841a0b2d7941f944b8 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:15:08 +0300 Subject: [PATCH 07/66] fix: s3 storage provider matching keys handling --- .../storageProviders/s3StorageProvider.ts | 75 ++++++++++++++----- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 3f106cc..02e9250 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -2,6 +2,7 @@ import { DeleteObjectsCommand, HeadBucketCommand, + HeadObjectCommand, ListObjectsV2Command, NoSuchBucket, NotFound, @@ -72,13 +73,13 @@ export class S3StorageProvider implements IStorageProvider path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion + + const exists = await this.bucketExists(bucket); if (!exists) { throw new UnrecoverableError(`Bucket does not exist: ${bucket}`); } - if (paths.some((path) => path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion - for (const path of paths) { const pathFailures = await this.deleteResource({ bucket, path }); failures.push(...pathFailures); @@ -114,29 +115,37 @@ export class S3StorageProvider implements IStorageProvider { return { path, reason }; }); } } - private async deleteResource({ bucket, path }: { path: string; bucket: string }): Promise { + private async deleteResource({ bucket, path }: { bucket: string; path: string }): Promise { const failures: DeleteFailure[] = []; - const normalizedPrefix = this.normalizePrefix(path); let totalDeletedObjectsCount = 0, totalFailedObjectsCount = 0; const s3Objects = this.getS3Objects({ bucket, - prefix: normalizedPrefix, + prefix: path, pageSize: this.s3Config.delete.batchSize, }); try { for await (const pageOfObjects of s3Objects) { - this.logger.debug(`Received a batch of ${pageOfObjects.length} objects to delete`); - const keys = pageOfObjects.map((obj) => obj.Key!).filter(Boolean); + this.logger.debug({ + msg: `Received a batch of ${pageOfObjects.length} objects to delete`, + pageOfObjects, + path, + pageSize: this.s3Config.delete.batchSize, + }); + const keys = pageOfObjects.map((obj) => obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); + + if (keys.length === 0) { + continue; + } const chunkFailures = await this.deleteChunk(keys, bucket); failures.push(...chunkFailures); @@ -147,20 +156,20 @@ export class S3StorageProvider implements IStorageProvider 0) this.logger.debug({ - msg: `Could not delete ${failedObjectsCount} objects out of total of ${totalFailedObjectsCount} objects that could not be deleted`, + msg: `Could not delete ${failedObjectsCount} objects. Totally ${totalFailedObjectsCount} objects could not be deleted`, }); } - this.logger.debug({ msg: 'Deletion completed', prefix: normalizedPrefix, totalDeletedObjectsCount, totalFailedObjectsCount }); + this.logger.debug({ msg: 'Deletion completed', path, totalDeletedObjectsCount, totalFailedObjectsCount }); return failures; } catch (err) { this.logger.error({ msg: 'Stream of objects was interrupted by an error', - prefix: normalizedPrefix, + path, totalDeletedObjectsCount, totalFailedObjectsCount, err, @@ -189,8 +198,13 @@ export class S3StorageProvider implements IStorageProvider { + try { + await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); + return true; + } catch (err) { + if (err instanceof NotFound) { + return false; + } else if (err instanceof NoSuchBucket) { + this.logger.warn({ msg: `S3 Error [${err.name}] no such bucket: ${err.message} (Req ID: ${err.$metadata.requestId})` }); + return false; + } else { + this.logger.error({ msg: 'resourceExists object check failed', err, bucket, path }); + throw err; + } + } + } + + // A key belongs to the target if it IS the target object or lives under 'target/'. + // The 'target/' guard prevents matching sibling keys that merely share the prefix + // (e.g. target 'photos' must not match 'photos_old/img.jpg', target 'metadata.txt' + // must not match 'metadata.txt.bak'). + private matchesTarget(key: string, target: string): boolean { + return key === target || key.startsWith(this.normalizePrefix(target)); } - private async storageExists(bucket: string): Promise { + private async bucketExists(bucket: string): Promise { try { this.logger.debug({ msg: `Checking bucket exists`, bucket }); const command = new HeadBucketCommand({ Bucket: bucket }); @@ -223,7 +262,7 @@ export class S3StorageProvider implements IStorageProvider Date: Mon, 20 Jul 2026 10:18:31 +0300 Subject: [PATCH 08/66] test: reorder and fix flaky tests and improve coverage --- .../s3StorageProvider.spec.ts | 282 +++++++++++++----- 1 file changed, 214 insertions(+), 68 deletions(-) diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 4b5e1f0..e8eede1 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -7,6 +7,7 @@ import { paginateListObjectsV2, S3Client, S3ServiceException, + type DeleteObjectsCommandOutput, } from '@aws-sdk/client-s3'; import { faker } from '@faker-js/faker'; import type { Logger } from '@map-colonies/js-logger'; @@ -254,42 +255,10 @@ describe('S3StorageProvider', () => { const PATH = 'layer/v1'; const NORMALIZED_PATH = `${PATH}/`; - it('should throw UnrecoverableError when bucket does not exist', async () => { - mockSend.mockRejectedValueOnce(new NotFound({ $metadata: {}, message: '' })); - - const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - - await expect(result).rejects.toThrow(UnrecoverableError); - }); - - it('should throw UnrecoverableError when storage check failing', async () => { - const expectedError = new Error('error'); - mockSend.mockRejectedValueOnce(expectedError); - - const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - - await expect(result).rejects.toThrow(expectedError); - }); - - it('should throw UnrecoverableError when a path is empty (root deletion guard)', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists - - const result = provider.deleteResources({ paths: [''], bucket: BUCKET, storageProvider: 'S3' }); - - await expect(result).rejects.toThrow(UnrecoverableError); - expect(paginateListObjectsV2).not.toHaveBeenCalled(); - }); - - it('should throw UnrecoverableError when any path is empty even if others are valid', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists - - const result = provider.deleteResources({ paths: [PATH, ''], bucket: BUCKET, storageProvider: 'S3' }); - - await expect(result).rejects.toThrow(UnrecoverableError); - }); - it('should return empty result when listing returns no keys', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -298,40 +267,68 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); - it('should normalize prefix by appending trailing slash', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists + it('should not double-add trailing slash when prefix already ends with one', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); - await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + await provider.deleteResources({ paths: [`${PATH}/`], bucket: BUCKET, storageProvider: 'S3' }); expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ Prefix: NORMALIZED_PATH })); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); - it('should not double-add trailing slash when prefix already ends with one', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists + it('should list and delete a single object', async () => { + const path = 'layer/v1/0/0.png'; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); - await provider.deleteResources({ paths: [`${PATH}/`], bucket: BUCKET, storageProvider: 'S3' }); + const result = await provider.deleteResources({ paths: [path], bucket: BUCKET, storageProvider: 'S3' }); - expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ Prefix: NORMALIZED_PATH })); + expect(result).toEqual({ failures: [] }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); it('should list and delete a single page of objects', async () => { const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 mockPaginateListObjectsV2Next .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete a single page of objects - no matching single object (bucket does not exists)', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; mockSend .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NoSuchBucket({ $metadata: {}, message: 'no bucket' })) // no bucket found for single object .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -339,20 +336,42 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); - expect(mockSend).toHaveBeenCalledTimes(2); + expect(mockSend).toHaveBeenCalledTimes(3); }); it('should list and delete multiple pages of objects', async () => { const page1Keys = ['layer/v1/0/0.png']; const page2Keys = ['layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 mockPaginateListObjectsV2Next .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should list and delete multiple pages of objects skipping deletion of empty keys', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; mockSend .mockResolvedValueOnce(undefined) // bucket exists - .mockResolvedValueOnce({ Errors: [] }) // delete page 1 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map(() => ({ Key: undefined })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -363,6 +382,28 @@ describe('S3StorageProvider', () => { expect(mockSend).toHaveBeenCalledTimes(3); }); + it('should list and a single object skipping deletion of similar keys of objects', async () => { + const objectKey = 'layer/v1/0/0.png'; + const page1Keys = [`${objectKey}8`]; + const page2Keys = [objectKey]; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [objectKey], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + it('should list and delete multiple paths and their objects', async () => { const PATH1 = 'layer/v1'; const PATH2 = 'layer/v2'; @@ -370,6 +411,14 @@ describe('S3StorageProvider', () => { const path1Page2Keys = ['layer/v1/0/1.png']; const path2Page1Keys = ['layer/v2/0/0.png']; const path2Page2Keys = ['layer/v2/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object for path 1 + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 1 + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 2 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object for path 2 + .mockResolvedValueOnce({ Errors: [] }) // delete path 2 page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete path 2 page 2 mockPaginateListObjectsV2Next .mockResolvedValueOnce({ done: false, value: { Contents: path1Page1Keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: false, value: { Contents: path1Page2Keys.map((Key) => ({ Key })) } }) @@ -377,12 +426,6 @@ describe('S3StorageProvider', () => { .mockResolvedValueOnce({ done: false, value: { Contents: path2Page1Keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: false, value: { Contents: path2Page2Keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: true, value: undefined }); - mockSend - .mockResolvedValueOnce(undefined) // bucket exists - .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 1 - .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 2 - .mockResolvedValueOnce({ Errors: [] }) // delete path 2 page 1 - .mockResolvedValueOnce({ Errors: [] }); // delete path 2 page 2 const result = await provider.deleteResources({ paths: [PATH1, PATH2], bucket: BUCKET, storageProvider: 'S3' }); @@ -390,14 +433,34 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(6); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); - expect(mockSend).toHaveBeenCalledTimes(5); + expect(mockSend).toHaveBeenCalledTimes(7); }); - it('should handle empty page Contents response', async () => { - mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: false, value: {} }).mockResolvedValueOnce({ done: true, value: undefined }); + it('should list and delete a single object and matching paths and their objects', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; mockSend .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce(undefined) // single object exists + .mockResolvedValueOnce({ Errors: [] }) // delete single object .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should handle empty page Contents response', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: false, value: {} }).mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -410,12 +473,32 @@ describe('S3StorageProvider', () => { it('should handle empty delete page Errors response', async () => { const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ $metadata: {} } satisfies DeleteObjectsCommandOutput); // delete page 1 mockPaginateListObjectsV2Next .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: [] }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should handle delete page Errors response without elements', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; mockSend .mockResolvedValueOnce(undefined) // bucket exists - .mockResolvedValueOnce({}); // delete page 1 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ $metadata: {}, Errors: [] } satisfies DeleteObjectsCommandOutput); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -423,20 +506,70 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should throw UnrecoverableError when bucket does not exist', async () => { + mockSend.mockRejectedValueOnce(new NotFound({ $metadata: {}, message: '' })); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it('should throw an error when storage existence check failing', async () => { + const expectedError = new Error('error'); + mockSend.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it('should throw UnrecoverableError when a path is empty (root deletion guard)', async () => { + const result = provider.deleteResources({ paths: [''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(paginateListObjectsV2).not.toHaveBeenCalled(); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + + it('should throw UnrecoverableError when any path is empty even if others are valid', async () => { + const result = provider.deleteResources({ paths: [PATH, ''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + + it('should throw an error when checking for matching single object is failing (generic error)', async () => { + const expectedError = new Error('NetworkError'); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(expectedError); // error thrown for single object lookup + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); expect(mockSend).toHaveBeenCalledTimes(2); }); it('should paginate and return all delete errors returned by S3 per object', async () => { const page1Keys = ['layer/v1/0/0.png']; const page2Keys = ['layer/v1/0/1.png']; - mockPaginateListObjectsV2Next - .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) - .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) - .mockResolvedValueOnce({ done: true, value: undefined }); mockSend .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete page 1 .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -444,20 +577,21 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); - expect(mockSend).toHaveBeenCalledTimes(3); + expect(mockSend).toHaveBeenCalledTimes(4); }); it('should paginate and return all delete errors thrown and unhandled by S3', async () => { const page1Keys = ['layer/v1/0/0.png']; const page2Keys = ['layer/v1/0/1.png']; - mockPaginateListObjectsV2Next - .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) - .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) - .mockResolvedValueOnce({ done: true, value: undefined }); mockSend .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object .mockRejectedValueOnce(new Error('NetworkError')) // delete page 1 .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -465,12 +599,14 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); - expect(mockSend).toHaveBeenCalledTimes(3); + expect(mockSend).toHaveBeenCalledTimes(4); }); it('should stop pagination and throw an error when bucket does not exist', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists const expectedError = new NoSuchBucket({ $metadata: { requestId: faker.string.uuid() }, message: 'msg' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); const result = provider.deleteResources({ paths: [PATH], bucket: '', storageProvider: 'S3' }); @@ -479,11 +615,14 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); it('should stop pagination and throw an error when S3 throws a service error', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists const expectedError = new S3ServiceException({ $fault: 'server', $metadata: { requestId: faker.string.uuid() }, name: 'msg' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -492,11 +631,14 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); it('should stop pagination and throw an error when S3 throws a bucket does not exists error', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists const expectedError = new NoSuchBucket({ $metadata: {}, message: '' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); @@ -505,12 +647,15 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); }); it('should stop pagination and return failure when list call throws', async () => { - mockSend.mockResolvedValueOnce(undefined); // bucket exists const expectedError = new Error('NetworkError'); const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object mockPaginateListObjectsV2Next .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) .mockRejectedValueOnce(expectedError); @@ -521,6 +666,7 @@ describe('S3StorageProvider', () => { expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); }); }); From fd3b1e39f6970e9c8b4f292bd87aaf982d2fd9e4 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:59:46 +0300 Subject: [PATCH 09/66] refactor: extract function to a path module --- src/cleaner/storageProviders/fsStorageProvider.ts | 4 ++-- src/cleaner/storageProviders/s3StorageProvider.ts | 10 +++------- src/cleaner/utils/index.ts | 1 + src/cleaner/utils/path.ts | 3 +++ 4 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 src/cleaner/utils/path.ts diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index e6f7a48..949e2d4 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -4,9 +4,9 @@ import { join, resolve, sep } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; -import { getChunk } from '../utils'; type FSStorageProviderType = Extract; @@ -123,7 +123,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private checkPathTraversal(paths: string[]): boolean { return paths.every((path) => { const absolutePath = this.resolveAbsolutePath(join(this.basePath, path)); - return absolutePath.startsWith(this.basePath); + return absolutePath.startsWith(normalizeFolderPath(this.basePath)); }); } diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 02e9250..ede30bc 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -15,8 +15,8 @@ import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { ConfigType } from '@common/config'; import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; import { describeError, UnrecoverableError } from '../errors'; -import { getChunk } from '../utils'; type S3StorageProviderType = Extract; @@ -89,7 +89,7 @@ export class S3StorageProvider implements IStorageProvider { - const prefix = this.normalizePrefix(relativePath); + const prefix = normalizeFolderPath(relativePath); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); return (result.KeyCount ?? 0) > 0; @@ -220,10 +220,6 @@ export class S3StorageProvider implements IStorageProvider { try { await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); @@ -246,7 +242,7 @@ export class S3StorageProvider implements IStorageProvider { diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index 574dfb3..fde0b66 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,3 +1,4 @@ export { getChunk } from './chunk'; export { buildPollingPairs } from './pairBuilder'; export { validateSchema } from './validationHelper'; +export { normalizeFolderPath } from './path'; diff --git a/src/cleaner/utils/path.ts b/src/cleaner/utils/path.ts new file mode 100644 index 0000000..68df844 --- /dev/null +++ b/src/cleaner/utils/path.ts @@ -0,0 +1,3 @@ +export const normalizeFolderPath = (path: string): string => { + return path.endsWith('/') ? path : `${path}/`; +}; From 7c4394175810d89534e00653907a94f7e48d9403 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:00:49 +0300 Subject: [PATCH 10/66] refactor: extract function to a path module --- .../storageProviders/fsStorageProvider.ts | 23 ++++--------------- src/cleaner/utils/index.ts | 2 +- src/cleaner/utils/path.ts | 15 +++++++++++- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 949e2d4..629f3db 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -1,10 +1,10 @@ import { accessSync, constants, statSync } from 'node:fs'; import { rm, rmdir, stat, unlink } from 'node:fs/promises'; -import { join, resolve, sep } from 'node:path'; +import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; -import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; +import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; @@ -26,7 +26,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private readonly logger: Logger ) { this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; - this.basePath = this.resolveAbsolutePath(this.fsConfig.basePath); + this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); this.logger.debug(`Using ${this.basePath} as base path for FS`); } @@ -71,10 +71,8 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, }: Extract): Promise { - // prevent path traversal (i.e. accessing folders above root folder) + // Prevent path traversal (i.e. accessing folders above root folder) if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); - if (!paths.every((path) => this.resolveAbsolutePath(join(this.basePath, path)) !== this.basePath)) - throw new UnrecoverableError(`Cannot delete base path itself`); const failures: DeleteFailure[] = []; for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { @@ -122,7 +120,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private checkPathTraversal(paths: string[]): boolean { return paths.every((path) => { - const absolutePath = this.resolveAbsolutePath(join(this.basePath, path)); + const absolutePath = resolveAbsolutePath(join(this.basePath, path)); return absolutePath.startsWith(normalizeFolderPath(this.basePath)); }); } @@ -169,15 +167,4 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { await Promise.allSettled([...dirs].map(async (dir) => rmdir(dir))); } } - - /** - * Resolves a file system path to an absolute path. - * Ensures the path is resolved as an absolute path and properly formatted - * with a leading separator if not already present. - * @param path - The input path string to normalize - * @returns An absolute path with proper path separators - */ - private resolveAbsolutePath(path: string): string { - return resolve(`${path.startsWith(sep) ? '' : sep}${path}`); - } } diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index fde0b66..9d2809c 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,4 +1,4 @@ export { getChunk } from './chunk'; export { buildPollingPairs } from './pairBuilder'; export { validateSchema } from './validationHelper'; -export { normalizeFolderPath } from './path'; +export { normalizeFolderPath, resolveAbsolutePath } from './path'; diff --git a/src/cleaner/utils/path.ts b/src/cleaner/utils/path.ts index 68df844..c3049ee 100644 --- a/src/cleaner/utils/path.ts +++ b/src/cleaner/utils/path.ts @@ -1,3 +1,16 @@ +import { resolve, sep } from 'node:path/posix'; + export const normalizeFolderPath = (path: string): string => { - return path.endsWith('/') ? path : `${path}/`; + return path.endsWith(sep) ? path : `${path}${sep}`; +}; + +/** + * Resolves a file system path to an absolute path. + * Ensures the path is resolved as an absolute path and properly formatted + * with a leading separator if not already present. + * @param path - The input path string to normalize + * @returns An absolute path with proper path separators + */ +export const resolveAbsolutePath = (path: string): string => { + return resolve(`${path.startsWith(sep) ? '' : sep}${path}`); }; From 9e0fe7feaf9ae81110a138dde02ccc979abac5a4 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:05:37 +0300 Subject: [PATCH 11/66] fix: fs and s3 bacth size config value parsing --- config/custom-environment-variables.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 98c242d..0686d21 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -80,7 +80,10 @@ }, "s3": { "delete": { - "batchSize": "S3_DELETE_BATCH_SIZE" + "batchSize": { + "__name": "S3_DELETE_BATCH_SIZE", + "__format": "number" + } }, "endpoint": "S3_ENDPOINT", "accessKeyId": "S3_ACCESS_KEY_ID", @@ -97,7 +100,10 @@ }, "fs": { "delete": { - "batchSize": "FS_DELETE_BATCH_SIZE" + "batchSize": { + "__name": "FS_DELETE_BATCH_SIZE", + "__format": "number" + } }, "basePath": "FS_BASE_PATH" } From a95b43b9c4f4c18e37f1ff7fd86e5467802fed03 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:53:54 +0300 Subject: [PATCH 12/66] fix: regression in delete of chunks exceeding 1000 objects --- src/cleaner/storageProviders/s3StorageProvider.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index ede30bc..e532ee4 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -20,6 +20,9 @@ import { describeError, UnrecoverableError } from '../errors'; type S3StorageProviderType = Extract; +// S3/MinIO reject DeleteObjects requests with more than 1000 keys, regardless of configured batch size. +const S3_DELETE_OBJECTS_MAX_KEYS = 1000; + export interface S3Config { delete: { batchSize: number; @@ -101,6 +104,18 @@ export class S3StorageProvider implements IStorageProvider { + const failures: DeleteFailure[] = []; + + // The configured batch size may exceed the DeleteObjects API limit, so re-chunk defensively here. + for (const keys of getChunk(paths, S3_DELETE_OBJECTS_MAX_KEYS)) { + const chunkFailures = await this.deleteObjects(keys, bucket); + failures.push(...chunkFailures); + } + + return failures; + } + + private async deleteObjects(paths: string[], bucket: string): Promise { try { const command = new DeleteObjectsCommand({ Bucket: bucket, From c19b1373842db8a7117d79007b4450b174759f21 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:58:21 +0300 Subject: [PATCH 13/66] test: add tests covering s3 chunk limit regression --- .../storageProviders/s3StorageProvider.spec.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index e8eede1..47a073c 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -169,6 +169,23 @@ describe('S3StorageProvider', () => { expect(secondCallInput.Delete.Objects).toHaveLength(500); }); + it('should cap DeleteObjects requests at S3 max keys limit (1000 keys) even when configured batchSize is larger', async () => { + const paths = Array.from({ length: 2500 }, (_, i) => `object-${i}.txt`); + const mockConfig = createMockS3Config({ + delete: { + batchSize: 2000, + }, + }); + provider = new S3StorageProvider(mockConfig, mockLogger); + + await provider.delete(paths, BUCKET); + + expect(mockSend).toHaveBeenCalledTimes(3); + const callInputs = vi.mocked(DeleteObjectsCommand).mock.calls.map((call) => call[0] as { Delete: { Objects: { Key: string }[] } }); + expect(callInputs.every((input) => input.Delete.Objects.length <= 1000)).toBe(true); + expect(callInputs.map((input) => input.Delete.Objects.length)).toEqual([1000, 1000, 500]); + }); + it('should accumulate failures across multiple chunks', async () => { const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); mockSend From 2163115afaa8fbf73126deaa53da05d6a5cfd724 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:59:26 +0300 Subject: [PATCH 14/66] fix: prevent batch size from being zero or less --- src/cleaner/storageProviders/fsStorageProvider.ts | 1 + src/cleaner/storageProviders/s3StorageProvider.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 629f3db..d625f91 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -26,6 +26,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private readonly logger: Logger ) { this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; + if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); this.logger.debug(`Using ${this.basePath} as base path for FS`); diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index e532ee4..f48525b 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -16,7 +16,7 @@ import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { ConfigType } from '@common/config'; import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; -import { describeError, UnrecoverableError } from '../errors'; +import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; type S3StorageProviderType = Extract; @@ -44,6 +44,7 @@ export class S3StorageProvider implements IStorageProvider Date: Mon, 20 Jul 2026 14:11:10 +0300 Subject: [PATCH 15/66] test: cover batch size configuration --- tests/storageProviders/fsStorageProvider.spec.ts | 7 +++++++ tests/storageProviders/s3StorageProvider.spec.ts | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index 568d4e5..ccc51ab 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -7,6 +7,7 @@ import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; import { FsStorageProvider, type FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; import type { ConfigType } from '@src/common/config'; import { createMockFsConfig, createMockLogger, FS_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; +import { faker } from '@faker-js/faker'; vi.mock('node:fs/promises', () => ({ stat: vi.fn(), @@ -304,5 +305,11 @@ describe('FsStorageProvider', () => { expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const zeroBatchConfig = createMockFsConfig({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => new FsStorageProvider(zeroBatchConfig, mockLogger)).toThrow(ConfigurationError); + }); }); }); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 47a073c..ade8544 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -12,7 +12,7 @@ import { import { faker } from '@faker-js/faker'; import type { Logger } from '@map-colonies/js-logger'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { UnrecoverableError } from '@src/cleaner/errors'; +import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; import type { ConfigType } from '@src/common/config'; import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; @@ -698,5 +698,11 @@ describe('S3StorageProvider', () => { }) ); }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const zeroBatchConfig = createMockS3Config({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => new S3StorageProvider(zeroBatchConfig, mockLogger)).toThrow(ConfigurationError); + }); }); }); From cd65186bcbcb6c6df7cc738baa9c3e35314e1c01 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:57:41 +0300 Subject: [PATCH 16/66] refactor: reduce number of potential matching paths --- .../storageProviders/s3StorageProvider.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index f48525b..e563e6d 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -203,16 +203,6 @@ export class S3StorageProvider implements IStorageProvider { - const paginatorConfig = { - client: this.s3Client, - pageSize, - }; - - const commandInput = { - Bucket: bucket, - Prefix: prefix, - }; - try { // First, if object exists it is removed. This is to mitigate an issue in MinIO that shadows paths sharing common path with an object. // Second, objects having this path are iterated and removed @@ -220,6 +210,16 @@ export class S3StorageProvider implements IStorageProvider Date: Mon, 20 Jul 2026 15:04:40 +0300 Subject: [PATCH 17/66] refactor: unshadow detected file instead of dir config error --- src/cleaner/storageProviders/fsStorageProvider.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index d625f91..32b81d8 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -115,6 +115,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { throw new ConfigurationError(`FS path exists but it is a file, not a directory: ${path}`); } } catch (err) { + if (err instanceof ConfigurationError) throw err; throw new ConfigurationError(`An unexpected error occurred on FS info check: ${describeError(err)}`); } } From 919c01af07e532882999408b445290c654305e9e Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:07:47 +0300 Subject: [PATCH 18/66] chore: missing env value in configmap --- helm/templates/configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index a5700d7..c5add3e 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -55,6 +55,7 @@ data: {{- with .Values.env.jobnik.worker }} JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }} {{- end }} + CLEANUP_STORAGE_PROVIDERS: {{ $storage.cleanupStorageProviders | toJson | quote }} {{- if has "S3" $storage.cleanupStorageProviders }} S3_DELETE_BATCH_SIZE: {{ $s3.delete.batchSize | default 1000 | quote }} S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }} From ba892c6c46932bb5c56a94eaa8ad4dd8ae528cc1 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:41:10 +0300 Subject: [PATCH 19/66] refactor: change fsBasePath from tilesDeletion strategy config --- config/custom-environment-variables.json | 1 - config/default.json | 1 - helm/templates/configmap.yaml | 1 - src/cleaner/storageProviders/index.ts | 2 +- src/cleaner/strategies/tilesDeletionStrategy.ts | 3 ++- tests/helpers/mocks.ts | 2 +- 6 files changed, 4 insertions(+), 6 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 0686d21..86c410e 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -111,7 +111,6 @@ "strategies": { "tilesDeletion": { "s3Bucket": "TILES_DELETION_S3_BUCKET", - "fsBasePath": "TILES_DELETION_FS_BASE_PATH", "batchSize": { "__name": "TILES_DELETION_BATCH_SIZE", "__format": "number" diff --git a/config/default.json b/config/default.json index 999d024..2dd1972 100644 --- a/config/default.json +++ b/config/default.json @@ -103,7 +103,6 @@ "strategies": { "tilesDeletion": { "s3Bucket": "", - "fsBasePath": "/tiles", "batchSize": 1000, "concurrency": 10, "failureSampleSize": 3 diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index c5add3e..1860c39 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -68,7 +68,6 @@ data: FS_BASE_PATH: {{ $tilesFSBasePath | quote }} {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} - TILES_DELETION_FS_BASE_PATH: {{ $tilesFSBasePath | quote }} {{- with .Values.env.strategies.tilesDeletion }} TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }} TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }} diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 5450fa1..4fb7851 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,4 @@ export { summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; -export { FsStorageProvider } from './fsStorageProvider'; +export { FsStorageProvider, type FsConfig } from './fsStorageProvider'; export type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index a151a04..e760e62 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -8,6 +8,7 @@ import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; import { summarizeDeleteFailures, type DeleteFailure, + type FsConfig, type IStorageProvider, type StorageProvider, type StorageProviders, @@ -38,7 +39,7 @@ export class TilesDeletionStrategy implements ITaskStrategy this.concurrency = config.get('strategies.tilesDeletion.concurrency') as unknown as number; this.failureSampleSize = config.get('strategies.tilesDeletion.failureSampleSize') as unknown as number; this.s3Bucket = config.get('strategies.tilesDeletion.s3Bucket') as unknown as string; - this.fsBasePath = config.get('strategies.tilesDeletion.fsBasePath') as unknown as string; // TODO: merge with fs.basePath + this.fsBasePath = config.get('storage.fs.basePath') as unknown as FsConfig['basePath']; } public validate(params: unknown): TilesDeletionParams { diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 7c3f5aa..13d31c7 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -92,7 +92,7 @@ export function createMockStrategyConfig(overrides: Record = {} 'strategies.tilesDeletion.concurrency': TILES_DELETION_CONFIG_DEFAULTS.concurrency, 'strategies.tilesDeletion.failureSampleSize': TILES_DELETION_CONFIG_DEFAULTS.failureSampleSize, 'strategies.tilesDeletion.s3Bucket': TILES_DELETION_CONFIG_DEFAULTS.s3Bucket, - 'strategies.tilesDeletion.fsBasePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, + 'storage.fs.basePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, ...overrides, }; return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; From 271560b6316e253d4423200d476edc2990bacef9 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:45:49 +0300 Subject: [PATCH 20/66] chore: add artifacts-deletion simulation to the deletion script --- scripts/simulate-deletion.mjs | 159 ++++++++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 6 deletions(-) diff --git a/scripts/simulate-deletion.mjs b/scripts/simulate-deletion.mjs index 59ef463..5d8a19a 100644 --- a/scripts/simulate-deletion.mjs +++ b/scripts/simulate-deletion.mjs @@ -1,5 +1,5 @@ /** - * Simulation script for tiles-deletion tasks. + * Simulation script for tiles-deletion and artifacts-deletion tasks. * * Usage: * node scripts/simulate-deletion.mjs --provider S3 @@ -9,6 +9,15 @@ * node scripts/simulate-deletion.mjs --provider S3 --partial # multi-zoom partial deletion * node scripts/simulate-deletion.mjs --provider FS --real-tiles --source-tile scripts/tile_deletion_test.jpeg * node scripts/simulate-deletion.mjs --provider S3 --real-tiles --source-tile scripts/tile_deletion_test.jpeg --zooms 17,18,19,20 --tile-count 400 + * node scripts/simulate-deletion.mjs --provider S3 --artifacts-deletion + * node scripts/simulate-deletion.mjs --provider FS --artifacts-deletion --paths config,gpkg,reports + * + * --artifacts-deletion: simulate the Delete_Layer / artifacts-deletion task + * (DeleteStoredResourcesStrategy), which recursively deletes whole resource + * trees (e.g. gpkg, config, validation reports) rather than a tile range. + * --paths : comma-separated relative paths to seed & delete (default: config,gpkg,reports) + * Each path is seeded with a couple of nested fake files to prove the whole + * subtree — not just its top-level file — gets removed. * * --real-tiles: seed a real local tile file replicated across a multi-zoom grid. * --source-tile : local file to use as tile content for every seeded tile (required) @@ -42,7 +51,7 @@ * * Override defaults with env vars: * QUEUE_JOB_MANAGER_BASE_URL, S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, - * TILES_DELETION_S3_BUCKET, TILES_DELETION_FS_BASE_PATH, TILES_PATH, ZOOM, MIN_X, MAX_X, MIN_Y, MAX_Y, + * TILES_DELETION_S3_BUCKET, FS_BASE_PATH, TILES_PATH, ARTIFACTS_PATH, ZOOM, MIN_X, MAX_X, MIN_Y, MAX_Y, * SEED_CONCURRENCY (default: 50) */ @@ -69,6 +78,7 @@ Modes (mutually exclusive): --partial Seed tiles across 3 zoom levels; task only deletes a subset --real-tiles Use a real tile file instead of fake content --skip-seed Skip seeding; go straight to job creation (idempotent delete) + --artifacts-deletion Simulate the Delete_Layer / artifacts-deletion task (whole-resource deletion) Real-tiles options (require --real-tiles): --source-tile Local tile file to replicate across the grid (required) @@ -76,14 +86,18 @@ Real-tiles options (require --real-tiles): --zooms Zoom levels, comma-separated (default: 17,18,19,20) --tile-count Total tiles to seed across all zoom levels (default: 400) +Artifacts-deletion options (require --artifacts-deletion): + --paths Relative resource paths to seed & delete, comma-separated (default: config,gpkg,reports) + Env vars (all optional — pod ConfigMap values are used automatically): QUEUE_JOB_MANAGER_BASE_URL Job manager endpoint S3_ENDPOINT S3 endpoint URL S3_ACCESS_KEY_ID S3 access key S3_SECRET_ACCESS_KEY S3 secret key - TILES_DELETION_S3_BUCKET S3 bucket name - TILES_DELETION_FS_BASE_PATH FS base path for tile files + TILES_DELETION_S3_BUCKET S3 bucket name (tiles-deletion) / S3 bucket for artifacts-deletion + FS_BASE_PATH FS base path for stored files (tiles and artifacts) TILES_PATH Relative path prefix for tiles (default: simulate/layer/v1) + ARTIFACTS_PATH Relative base path for artifacts-deletion resources (default: simulate/artifacts) ZOOM Zoom level (default: 10) MIN_X, MAX_X X tile range (default: 0..3) MIN_Y, MAX_Y Y tile range (default: 0..3) @@ -94,6 +108,7 @@ Examples: node scripts/simulate-deletion.mjs --provider FS --partial node scripts/simulate-deletion.mjs --provider S3 --real-tiles --source-tile scripts/tile_deletion_test.jpeg node scripts/simulate-deletion.mjs --provider FS --real-tiles --source-tile scripts/tile_deletion_test.jpeg + node scripts/simulate-deletion.mjs --provider S3 --artifacts-deletion MAX_X=999 MAX_Y=999 ZOOM=18 node scripts/simulate-deletion.mjs --provider S3 --skip-seed `); process.exit(0); @@ -105,6 +120,7 @@ if (!providerFlag || !['S3', 'FS'].includes(providerFlag)) { console.error( ' node scripts/simulate-deletion.mjs --provider --real-tiles --source-tile [--zooms ] [--tile-count ]' ); + console.error(' node scripts/simulate-deletion.mjs --provider --artifacts-deletion [--paths ]'); console.error('\nRun with --help for full usage information.'); process.exit(1); } @@ -112,12 +128,16 @@ const PROVIDER = providerFlag; const SKIP_SEED = args.includes('--skip-seed'); const PARTIAL = args.includes('--partial'); const REAL_TILES = args.includes('--real-tiles'); +const ARTIFACTS_DELETION = args.includes('--artifacts-deletion'); // real-tiles specific args const SOURCE_TILE = args.includes('--source-tile') ? args[args.indexOf('--source-tile') + 1] : null; const ZOOMS_INPUT = args.includes('--zooms') ? args[args.indexOf('--zooms') + 1] : '17,18,19,20'; const TILE_COUNT = args.includes('--tile-count') ? Number(args[args.indexOf('--tile-count') + 1]) : 400; +// artifacts-deletion specific args +const ARTIFACT_PATHS_INPUT = args.includes('--paths') ? args[args.indexOf('--paths') + 1] : 'config,gpkg,reports'; + if (REAL_TILES) { if (!SOURCE_TILE) { console.error('--real-tiles requires --source-tile '); @@ -129,6 +149,11 @@ if (REAL_TILES) { } } +if (ARTIFACTS_DELETION && (PARTIAL || REAL_TILES)) { + console.error('--artifacts-deletion cannot be combined with --partial or --real-tiles'); + process.exit(1); +} + // ─── Read local.json as config base (env vars override) ────────────────────── let localConfig = {}; @@ -141,7 +166,8 @@ try { } const cfg = { - s3: localConfig.s3 ?? {}, + s3: localConfig.storage?.s3 ?? {}, + fs: localConfig.storage?.fs ?? {}, strategies: localConfig.strategies?.tilesDeletion ?? {}, queue: localConfig.queue ?? {}, }; @@ -154,7 +180,7 @@ const S3_ENDPOINT = process.env.S3_ENDPOINT ?? cfg.s3.endpoint ?? 'http://localh const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID ?? cfg.s3.accessKeyId ?? 'minioadmin'; const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY ?? cfg.s3.secretAccessKey ?? 'minioadmin'; const S3_BUCKET = process.env.TILES_DELETION_S3_BUCKET ?? cfg.strategies.s3Bucket ?? ''; -const FS_BASE_PATH = process.env.TILES_DELETION_FS_BASE_PATH ?? cfg.strategies.fsBasePath ?? '/tiles'; +const FS_BASE_PATH = process.env.FS_BASE_PATH ?? cfg.fs.basePath ?? '/tiles'; const SEED_CONCURRENCY = Number(process.env.SEED_CONCURRENCY ?? 200); // Tile range to seed + delete @@ -166,6 +192,11 @@ const MIN_Y = Number(process.env.MIN_Y ?? 0); const MAX_Y = Number(process.env.MAX_Y ?? 3); const FILE_EXTENSION = 'jpeg'; +// Artifacts-deletion resource paths to seed + delete +const ARTIFACTS_PATH = process.env.ARTIFACTS_PATH ?? 'simulate/artifacts'; +const ARTIFACT_SUBPATHS = ARTIFACT_PATHS_INPUT.split(',').map((s) => s.trim()); +const ARTIFACT_PATHS = ARTIFACT_SUBPATHS.map((subPath) => `${ARTIFACTS_PATH}/${subPath}`); + // ─── Partial-deletion scenario definition ──────────────────────────────────── // // Three zoom levels are seeded identically (4×4 grid by default). @@ -527,9 +558,125 @@ async function runRealTilesMode() { await createRealTilesJob(zooms, grid, TILES_PATH, ext); } +// ─── Artifacts-deletion mode (Delete_Layer / artifacts-deletion task) ───────── +// +// Unlike tiles-deletion (which deletes a tile-range grid), artifacts-deletion +// recursively removes whole resource trees given their root paths (e.g. a +// layer's gpkg, config and validation-report directories). Each seeded path +// gets a couple of nested files to prove the whole subtree is removed, not +// just its top-level entry. + +function* artifactFilesForPath(rootPath) { + yield `${rootPath}/file.dat`; + yield `${rootPath}/nested/file.dat`; +} + +async function seedS3Artifacts() { + if (!S3_BUCKET) { + throw new Error('S3_BUCKET env var is required for S3 provider (or set it in config/local.json)'); + } + + const client = new S3Client({ + endpoint: S3_ENDPOINT, + credentials: { accessKeyId: S3_ACCESS_KEY_ID, secretAccessKey: S3_SECRET_ACCESS_KEY }, + forcePathStyle: true, + region: 'us-east-1', + tls: false, + }); + + console.log(`[S3] Seeding artifacts under s3://${S3_BUCKET}/{${ARTIFACT_PATHS.join(', ')}}/...`); + for (const rootPath of ARTIFACT_PATHS) { + for (const key of artifactFilesForPath(rootPath)) { + await client.send( + new PutObjectCommand({ Bucket: S3_BUCKET, Key: key, Body: Buffer.from('fake-artifact'), ContentType: 'application/octet-stream' }) + ); + } + } + const firstKey = [...artifactFilesForPath(ARTIFACT_PATHS[0])][0]; + await client.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: firstKey })); + console.log(`[S3] Verified: s3://${S3_BUCKET}/${firstKey} exists`); + console.log(`[S3] Done — seeded ${ARTIFACT_PATHS.length} artifact path(s).\n`); +} + +function seedFsArtifacts() { + console.log(`[FS] Seeding artifacts under ${FS_BASE_PATH}/{${ARTIFACT_PATHS.join(', ')}}/...`); + for (const rootPath of ARTIFACT_PATHS) { + for (const relativePath of artifactFilesForPath(rootPath)) { + const fullPath = join(FS_BASE_PATH, relativePath); + mkdirSync(join(fullPath, '..'), { recursive: true }); + writeFileSync(fullPath, 'fake-artifact'); + } + } + const firstPath = join(FS_BASE_PATH, [...artifactFilesForPath(ARTIFACT_PATHS[0])][0]); + if (!existsSync(firstPath)) throw new Error(`Seeding failed — ${firstPath} not found`); + console.log(`[FS] Verified: ${firstPath} exists`); + console.log(`[FS] Done — seeded ${ARTIFACT_PATHS.length} artifact path(s).\n`); +} + +async function createArtifactsDeletionJob() { + const taskParameters = { + storageProvider: PROVIDER, + paths: ARTIFACT_PATHS, + ...(PROVIDER === 'S3' && { bucket: S3_BUCKET }), + }; + + const body = { + resourceId: `simulate-artifacts-${PROVIDER.toLowerCase()}-${Date.now()}`, + version: '1.0.0', + type: 'Delete_Layer', + parameters: {}, + domain: 'RASTER', + tasks: [{ type: 'artifacts-deletion', parameters: taskParameters }], + }; + + console.log('[Job] Creating job with task parameters:'); + console.log(JSON.stringify(taskParameters, null, 2)); + + const res = await fetch(`${JOB_MANAGER_URL}/jobs`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Job creation failed [${res.status}]: ${text}`); + } + + const { id: jobId, taskIds } = await res.json(); + + console.log(`\n[Job] Created successfully:`); + console.log(` Job ID : ${jobId}`); + console.log(` Task ID : ${taskIds[0]}`); + console.log(`\nStart the cleaner — it will pick up task "${taskIds[0]}" and delete ${ARTIFACT_PATHS.length} artifact path(s).`); + console.log(`Track progress: GET ${JOB_MANAGER_URL}/jobs/${jobId}?shouldReturnTasks=true`); +} + +async function runArtifactsDeletionMode() { + if (!SKIP_SEED) { + if (PROVIDER === 'S3') { + await seedS3Artifacts(); + } else { + seedFsArtifacts(); + } + } else { + console.log(`[seed] Skipped — missing paths are treated as success (idempotent delete).\n`); + } + + await createArtifactsDeletionJob(); +} + // ─── Main ───────────────────────────────────────────────────────────────────── async function main() { + if (ARTIFACTS_DELETION) { + console.log( + `\n=== Simulating artifacts-deletion | storageProvider: ${PROVIDER} | paths: ${ARTIFACT_PATHS.length} | seed: ${SKIP_SEED ? 'skipped' : 'yes'} ===\n` + ); + await runArtifactsDeletionMode(); + return; + } + if (REAL_TILES) { const zooms = parseZoomList(ZOOMS_INPUT); const ext = extname(SOURCE_TILE).slice(1).toLowerCase() || FILE_EXTENSION; From e0bbca79ed7e5660a9f99fb80a87963707e27919 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:56:15 +0300 Subject: [PATCH 21/66] test: fix lint error of import order --- tests/storageProviders/fsStorageProvider.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index ccc51ab..ecdb749 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -3,11 +3,11 @@ import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { faker } from '@faker-js/faker'; import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; import { FsStorageProvider, type FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; import type { ConfigType } from '@src/common/config'; import { createMockFsConfig, createMockLogger, FS_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; -import { faker } from '@faker-js/faker'; vi.mock('node:fs/promises', () => ({ stat: vi.fn(), From 398fb51d1b71aeba485e62f07fd655baa1bdfe54 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:47:03 +0300 Subject: [PATCH 22/66] refactor: reorder functions --- .../storageProviders/s3StorageProvider.ts | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index e563e6d..e7e4318 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -104,6 +104,24 @@ export class S3StorageProvider implements IStorageProvider { + try { + this.logger.debug({ msg: `Checking bucket exists`, bucket }); + const command = new HeadBucketCommand({ Bucket: bucket }); + await this.s3Client.send(command); // If it resolves, the bucket exists and you have permission to access it + this.logger.debug({ msg: `Bucket exists`, bucket }); + return true; + } catch (err) { + if (err instanceof NotFound) { + this.logger.error({ msg: `Bucket does not exist`, bucket, err }); + return false; + } + const reason = describeError(err); + this.logger.error({ msg: 'Failed to check if bucket exists', bucket, reason, err }); + throw err; + } + } + private async deleteChunk(paths: string[], bucket: string): Promise { const failures: DeleteFailure[] = []; @@ -236,23 +254,6 @@ export class S3StorageProvider implements IStorageProvider { - try { - await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); - return true; - } catch (err) { - if (err instanceof NotFound) { - return false; - } else if (err instanceof NoSuchBucket) { - this.logger.warn({ msg: `S3 Error [${err.name}] no such bucket: ${err.message} (Req ID: ${err.$metadata.requestId})` }); - return false; - } else { - this.logger.error({ msg: 'resourceExists object check failed', err, bucket, path }); - throw err; - } - } - } - // A key belongs to the target if it IS the target object or lives under 'target/'. // The 'target/' guard prevents matching sibling keys that merely share the prefix // (e.g. target 'photos' must not match 'photos_old/img.jpg', target 'metadata.txt' @@ -261,21 +262,20 @@ export class S3StorageProvider implements IStorageProvider { + private async resourceExists({ bucket, path }: { bucket: string; path: string }): Promise { try { - this.logger.debug({ msg: `Checking bucket exists`, bucket }); - const command = new HeadBucketCommand({ Bucket: bucket }); - await this.s3Client.send(command); // If it resolves, the bucket exists and you have permission to access it - this.logger.debug({ msg: `Bucket exists`, bucket }); + await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); return true; } catch (err) { if (err instanceof NotFound) { - this.logger.error({ msg: `Bucket does not exist`, bucket, err }); return false; + } else if (err instanceof NoSuchBucket) { + this.logger.warn({ msg: `S3 Error [${err.name}] no such bucket: ${err.message} (Req ID: ${err.$metadata.requestId})` }); + return false; + } else { + this.logger.error({ msg: 'resourceExists object check failed', err, bucket, path }); + throw err; } - const reason = describeError(err); - this.logger.error({ msg: 'Failed to check if bucket exists', bucket, reason, err }); - throw err; } } } From 96dd83b4d0bcb15266de14911a4517c31503a876 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:05:54 +0300 Subject: [PATCH 23/66] refactor: modify providers deletion response --- src/cleaner/storageProviders/iStorageProvider.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index f798f12..99c5fe5 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -1,16 +1,12 @@ import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; /** - * A single failed deletion paired with a short reason string (e.g. 'ENOENT', - * 'AccessDenied', 'NoSuchKey'). + * A storage for failures with additional metadata. */ -export interface DeleteFailure { - path: string; - reason: string; -} +export type DeleteFailure = Map; export interface DeleteResourcesResult { - failures: DeleteFailure[]; + failures: DeleteFailure; } export type StorageProvider = DeleteStoredResourcesParams['storageProvider']; @@ -21,10 +17,10 @@ export interface IStorageProvider { * - S3: storageTarget = bucket name; paths are object keys * - FS: storageTarget = base directory; full path = join(storageTarget, path) * - * Returns one entry per failed deletion. "Not found" is reported as a failure - * (with reason 'ENOENT' / 'NoSuchKey'). + * Returns an object including delete failures aggregation with one entry per failed reason. + * "Not found" is reported as a failure with additional metadata on failure - count and sample */ - delete: (paths: string[], storageTarget: string) => Promise; + delete: (paths: string[], storageTarget: string) => Promise; /** * Deletes ALL objects/files under the given paths. From a4990067af58314998fb0194033ed3c3ccbf93ca Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:08:00 +0300 Subject: [PATCH 24/66] refactor: modify deletion failure summary report --- .../storageProviders/deleteFailureSummary.ts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/cleaner/storageProviders/deleteFailureSummary.ts b/src/cleaner/storageProviders/deleteFailureSummary.ts index 2cd7aad..00d8674 100644 --- a/src/cleaner/storageProviders/deleteFailureSummary.ts +++ b/src/cleaner/storageProviders/deleteFailureSummary.ts @@ -1,33 +1,31 @@ -import type { DeleteFailure } from './iStorageProvider'; +import type { DeleteResourcesResult } from './iStorageProvider'; /** * Aggregated view of a batch of delete failures — designed to be embedded in a * task rejection reason or log line without further post-processing. */ export interface DeleteFailureSummary { - /** Raw count per reason string, e.g. `{ ENOENT: 150, EACCES: 3 }`. */ - counts: Record; + /** Count of all failures */ + failuresCount: number; /** Reasons formatted descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ summary: string; - /** Up to `sampleSize` `path (reason)` strings, preserving input order. */ - sample: string[]; } /** * Reduces a list of provider delete failures into a compact, log-friendly * shape. Lives alongside `IStorageProvider` because it operates purely on - * `DeleteFailure[]` — any caller of `provider.delete()` can use it, regardless + * `DeleteResourcesResult` — any caller of `provider.delete()` can use it, regardless * of which storage backend produced the failures. */ -export function summarizeDeleteFailures(failures: DeleteFailure[], sampleSize: number): DeleteFailureSummary { - const counts: Record = {}; - for (const { reason } of failures) { - counts[reason] = (counts[reason] ?? 0) + 1; +export function summarizeDeleteFailures({ failures }: DeleteResourcesResult): DeleteFailureSummary { + let failuresCount = 0; + for (const { count } of failures.values()) { + failuresCount += count; } - const summary = Object.entries(counts) - .sort(([, a], [, b]) => b - a) - .map(([reason, count]) => `${reason}=${count}`) + + const summary = Object.entries(failures) + .sort(([, { count: a }], [, { count: b }]) => b - a) + .map(([reason, { count }]) => `${reason}=${count}`) .join(', '); - const sample = failures.slice(0, sampleSize).map((f) => `${f.path} (${f.reason})`); - return { counts, summary, sample }; + return { failuresCount, summary }; } From 2bb2ccb3ce86b7c89b12c97aeb2871a4429536fb Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:11:07 +0300 Subject: [PATCH 25/66] feat: add a function to merge failure results --- .../storageProviders/deleteFailureSummary.ts | 13 ++++++++++++- src/cleaner/storageProviders/index.ts | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/cleaner/storageProviders/deleteFailureSummary.ts b/src/cleaner/storageProviders/deleteFailureSummary.ts index 00d8674..0f40c1f 100644 --- a/src/cleaner/storageProviders/deleteFailureSummary.ts +++ b/src/cleaner/storageProviders/deleteFailureSummary.ts @@ -1,4 +1,4 @@ -import type { DeleteResourcesResult } from './iStorageProvider'; +import type { DeleteFailure, DeleteResourcesResult } from './iStorageProvider'; /** * Aggregated view of a batch of delete failures — designed to be embedded in a @@ -11,6 +11,17 @@ export interface DeleteFailureSummary { summary: string; } +export const mergeFailures = ({ source, target }: { source: DeleteFailure; target: DeleteFailure }): DeleteFailure => { + const failures: DeleteFailure = structuredClone(target); + + source.forEach((failed, reason) => { + const failure = failures.get(reason); + failures.set(reason, { count: (failure?.count ?? 0) + failed.count, sample: failure?.sample ?? failed.sample }); + }); + + return failures; +}; + /** * Reduces a list of provider delete failures into a compact, log-friendly * shape. Lives alongside `IStorageProvider` because it operates purely on diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 4fb7851..622d903 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,4 @@ -export { summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; +export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; export { FsStorageProvider, type FsConfig } from './fsStorageProvider'; export type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; From a10ab796462c64c858885a70da081800943d2941 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:24:59 +0300 Subject: [PATCH 26/66] refactor: update s3 storage provider with failure output change --- .../storageProviders/s3StorageProvider.ts | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index e7e4318..524fae1 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -9,12 +9,19 @@ import { paginateListObjectsV2, S3Client, S3ServiceException, + type _Error, type _Object, } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { ConfigType } from '@common/config'; -import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import { + mergeFailures, + type DeleteFailure, + type DeleteResourcesResult, + type IStorageProvider, + type StorageProvider, +} from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; @@ -57,25 +64,24 @@ export class S3StorageProvider implements IStorageProvider { + public async delete(paths: string[], bucket: string): Promise { this.logger.debug({ msg: 'Deleting objects from S3', bucket, count: paths.length }); + let failures: DeleteFailure = new Map(); - const failures: DeleteFailure[] = []; - - for (const chunk of getChunk(paths, this.s3Config.delete.batchSize)) { - const failed = await this.deleteChunk(chunk, bucket); - failures.push(...failed); + for (const chunk of getChunk(paths, Math.min(this.s3Config.delete.batchSize, S3_DELETE_OBJECTS_MAX_KEYS))) { + const chunkFailures = await this.deleteObjects(chunk, bucket); + failures = mergeFailures({ source: chunkFailures, target: failures }); } - return failures; + return { failures }; } public async deleteResources({ bucket, paths, }: Extract): Promise { - const failures: DeleteFailure[] = []; this.logger.debug({ msg: `Starting S3 deletion`, bucket, paths }); + let failures: DeleteFailure = new Map(); if (paths.some((path) => path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion @@ -86,7 +92,7 @@ export class S3StorageProvider implements IStorageProvider { - const failures: DeleteFailure[] = []; - - // The configured batch size may exceed the DeleteObjects API limit, so re-chunk defensively here. - for (const keys of getChunk(paths, S3_DELETE_OBJECTS_MAX_KEYS)) { - const chunkFailures = await this.deleteObjects(keys, bucket); - failures.push(...chunkFailures); - } - - return failures; - } - - private async deleteObjects(paths: string[], bucket: string): Promise { + private async deleteObjects(paths: string[], bucket: string): Promise { try { const command = new DeleteObjectsCommand({ Bucket: bucket, @@ -142,29 +136,33 @@ export class S3StorageProvider implements IStorageProvider Boolean(error.Key)) // Only include entries with a Key so every failure maps to a specific path. - .map((error) => ({ path: error.Key, reason: error.Code ?? error.Message ?? 'Unknown' })); - if (errors.length > 0) this.logger.warn({ msg: `Failed to delete ${errors.length} out of ${paths.length} objects` }); - return errors; + const failures: DeleteFailure = new Map(); + (response.Errors ?? []) + .filter((error): error is _Error & Required> => error.Key !== undefined) // Only include entries with a Key + .forEach((error) => { + const reason = error.Code ?? error.Message ?? 'Unknown'; + const failure = failures.get(reason); + failures.set(reason, { count: (failure?.count ?? 0) + 1, sample: error.Key }); + }); + + if (failures.size > 0) this.logger.warn({ msg: `Failed to delete ${failures.size} out of ${paths.length} objects` }); + return failures; } catch (err) { const reason = describeError(err); - this.logger.error({ msg: 'S3 batch request failed', bucket, reason, err }); - return paths.map((path) => { - return { path, reason }; - }); + this.logger.error({ msg: 'S3 delete objects request failed', bucket, reason, err }); + return new Map([[reason, { count: paths.length, sample: paths[0]! }]]); } } - private async deleteResource({ bucket, path }: { bucket: string; path: string }): Promise { - const failures: DeleteFailure[] = []; + private async deleteResource({ bucket, path }: { bucket: string; path: string }): Promise { + let failures: DeleteFailure = new Map(); let totalDeletedObjectsCount = 0, totalFailedObjectsCount = 0; const s3Objects = this.getS3Objects({ bucket, prefix: path, - pageSize: this.s3Config.delete.batchSize, + pageSize: Math.min(this.s3Config.delete.batchSize, S3_DELETE_OBJECTS_MAX_KEYS), }); try { @@ -181,10 +179,10 @@ export class S3StorageProvider implements IStorageProvider Date: Thu, 23 Jul 2026 10:52:09 +0300 Subject: [PATCH 27/66] refactor: update fs storage provider with failure output change --- .../storageProviders/fsStorageProvider.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 32b81d8..2f3ece4 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -7,6 +7,7 @@ import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StoragePro import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; +import { mergeFailures } from './deleteFailureSummary'; type FSStorageProviderType = Extract; @@ -42,10 +43,10 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } } - public async delete(paths: string[], storageTarget: string): Promise { + public async delete(paths: string[], storageTarget: string): Promise { this.logger.info({ msg: 'Deleting files from filesystem', basePath: storageTarget, count: paths.length }); + let failures: DeleteFailure = new Map(); - const failures: DeleteFailure[] = []; for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( relativePaths.map(async (relativePath) => { @@ -53,20 +54,23 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { }) ); + const chunkFailures: DeleteFailure = new Map(); for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { const relativePath = relativePaths[idx]!; const error: unknown = result.reason; const reason = describeError(error); this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); - failures.push({ path: relativePath, reason }); + const chunkFailure = chunkFailures.get(reason); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePaths[idx]! }); } } + failures = mergeFailures({ source: chunkFailures, target: failures }); } await this.cleanupEmptyDirs(paths, storageTarget); - return failures; + return { failures }; } public async deleteResources({ @@ -75,7 +79,8 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { // Prevent path traversal (i.e. accessing folders above root folder) if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); - const failures: DeleteFailure[] = []; + let failures: DeleteFailure = new Map(); + for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( relativePaths.map(async (relativePath) => { @@ -83,14 +88,17 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { }) ); + const chunkFailures: DeleteFailure = new Map(); for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { const fullPath = join(this.basePath, relativePaths[idx]!); const reason = describeError(result.reason); this.logger.error({ msg: 'Failed to delete layer directory', fullPath, reason, err: result.reason }); - failures.push({ path: fullPath, reason }); + const chunkFailure = chunkFailures.get(reason); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePaths[idx]! }); } } + failures = mergeFailures({ source: chunkFailures, target: failures }); } return { failures }; From 82fd062a27323ace1a121c3ab3f95e57a410cd40 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:53:48 +0300 Subject: [PATCH 28/66] refactor: update delete stored resoures strategy with failure output change --- .../strategies/deleteStoredResourcesStrategy.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts index 5b0aa09..fc66b20 100644 --- a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -10,15 +10,11 @@ import type { ITaskStrategy } from './taskStrategy'; @injectable() export class DeleteStoredResourcesStrategy implements ITaskStrategy { - private readonly failureSampleSize: number; - public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: StorageProviders - ) { - this.failureSampleSize = this.config.get('strategies.storedResourcesDeletion.failureSampleSize') as unknown as number; - } + ) {} public validate(params: unknown): DeleteStoredResourcesParams { this.logger.debug({ msg: `Validating input parameters` }); @@ -39,17 +35,16 @@ export class DeleteStoredResourcesStrategy implements ITaskStrategy 0) { - const { counts, summary, sample } = summarizeDeleteFailures(failures, this.failureSampleSize); + if (Object.keys(failures).length > 0) { + const { failuresCount, summary } = summarizeDeleteFailures({ failures }); this.logger.error({ msg: 'Deletion failed', provider: params.storageProvider, paths, - failureCount: failures.length, - reasonCounts: counts, - sample, + failuresCount, + failures, }); - throw new RecoverableError(`Failed to delete resource(s). Reasons: ${summary}. Sample: ${sample.join(', ')}`); + throw new RecoverableError(`Failed to delete ${failuresCount} objects. Reasons: ${summary}.`); } this.logger.info({ From 47310b8d2bee9628ddcfb7ce927a4ea1fae09ed6 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:08:29 +0300 Subject: [PATCH 29/66] refactor: update tiles deletion strategy with failure output change --- .../strategies/tilesDeletionStrategy.ts | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index e760e62..147df18 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -6,6 +6,7 @@ import { inject, injectable } from 'tsyringe'; import type { ConfigType } from '@common/config'; import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; import { + mergeFailures, summarizeDeleteFailures, type DeleteFailure, type FsConfig, @@ -75,36 +76,36 @@ export class TilesDeletionStrategy implements ITaskStrategy * matches the desired end-state) but counted separately for visibility. * Terminal progress to 100% is handled by the queue's task-ack — no explicit call needed here. */ - private reportOutcome(failures: DeleteFailure[], totalTiles: number): void { - const retryable: DeleteFailure[] = []; - const notFound: DeleteFailure[] = []; + private reportOutcome(failures: DeleteFailure, totalTiles: number): void { + const retryable: DeleteFailure = new Map(); + const notFound: DeleteFailure = new Map(); for (const failure of failures) { - (NOT_FOUND_REASONS.has(failure.reason) ? notFound : retryable).push(failure); + (NOT_FOUND_REASONS.has(failure[0]) ? notFound : retryable).set(failure[0], failure[1]); } - const deletedCount = totalTiles - retryable.length - notFound.length; + const deletedCount = totalTiles - retryable.size - notFound.size; - if (retryable.length > 0) { - const { counts, summary, sample } = summarizeDeleteFailures(retryable, this.failureSampleSize); + if (retryable.size > 0) { + const { failuresCount, summary } = summarizeDeleteFailures({ failures: retryable }); this.logger.error({ msg: 'Tiles deletion partially failed', totalTiles, - failedCount: retryable.length, - notFoundCount: notFound.length, + failedCount: retryable.size, + notFoundCount: notFound.size, deletedCount, - reasonCounts: counts, - sample, + failuresCount, + summary, }); - throw new RecoverableError(`Failed to delete ${retryable.length} tiles. Reasons: ${summary}. Sample: ${sample.join(', ')}`); + throw new RecoverableError(`Failed to delete ${retryable.size} tiles. Reasons: ${summary}.`); } - if (notFound.length > 0) { + if (notFound.size > 0) { this.logger.warn({ msg: 'Tiles deletion completed with missing tiles', totalTiles, - notFoundCount: notFound.length, + notFoundCount: notFound.size, deletedCount, - allTilesMissing: notFound.length === totalTiles, + allTilesMissing: notFound.size === totalTiles, }); return; } @@ -129,9 +130,9 @@ export class TilesDeletionStrategy implements ITaskStrategy storageTarget: string, params: TilesDeletionParams, totalTiles: number - ): Promise { + ): Promise { const { jobId, taskId } = this.taskContext; - const failures: DeleteFailure[] = []; + let failures: DeleteFailure = new Map(); const pendingBatches: string[][] = []; let batch: string[] = []; let processedTiles = 0; @@ -142,10 +143,12 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); batch = []; if (pendingBatches.length === this.concurrency) { - processedTiles += await this.flushBatches(provider, storageTarget, pendingBatches, failures); + const { batchFailures, processedTilesCount } = await this.flushBatches(provider, storageTarget, pendingBatches); + processedTiles += processedTilesCount; + failures = mergeFailures({ source: batchFailures, target: failures }); const percentage = Math.round((processedTiles / totalTiles) * PERCENTAGE_COMPLETE); await this.queueClient.updateProgress(jobId, taskId, percentage); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.size }); } } } @@ -154,8 +157,9 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); } if (pendingBatches.length > 0) { - await this.flushBatches(provider, storageTarget, pendingBatches, failures); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); + const { batchFailures } = await this.flushBatches(provider, storageTarget, pendingBatches); + failures = mergeFailures({ source: batchFailures, target: failures }); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.size }); } return failures; @@ -174,24 +178,26 @@ export class TilesDeletionStrategy implements ITaskStrategy private async flushBatches( provider: IStorageProvider, storageTarget: string, - pendingBatches: string[][], - failures: DeleteFailure[] - ): Promise { - const flushedCount = pendingBatches.reduce((sum, b) => sum + b.length, 0); + pendingBatches: string[][] + ): Promise<{ batchFailures: DeleteFailure; processedTilesCount: number }> { + let failures: DeleteFailure = new Map(); + + const processedTilesCount = pendingBatches.reduce((sum, b) => sum + b.length, 0); const results = await Promise.allSettled(pendingBatches.map(async (batch) => provider.delete(batch, storageTarget))); for (const [index, result] of results.entries()) { if (result.status === 'fulfilled') { - failures.push(...result.value); + failures = mergeFailures({ source: result.value.failures, target: failures }); } else { const error: unknown = result.reason; const reason = describeError(error); this.logger.error({ msg: 'Batch delete threw unexpectedly', reason, error }); const batch = pendingBatches[index] ?? []; - failures.push(...batch.map((path) => ({ path, reason }))); + const failure = failures.get(reason); + failures.set(reason, { count: (failure?.count ?? 0) + 1, sample: failure?.sample ?? batch[0]! }); } } pendingBatches.length = 0; - return flushedCount; + return { batchFailures: failures, processedTilesCount }; } /** From eaae536586d067c1803782bcc4aeecdf6026460b Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:24:30 +0300 Subject: [PATCH 30/66] refactor: rename file --- .../{deleteFailureSummary.ts => failuresHandling.ts} | 0 src/cleaner/storageProviders/fsStorageProvider.ts | 9 +++++++-- src/cleaner/storageProviders/index.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) rename src/cleaner/storageProviders/{deleteFailureSummary.ts => failuresHandling.ts} (100%) diff --git a/src/cleaner/storageProviders/deleteFailureSummary.ts b/src/cleaner/storageProviders/failuresHandling.ts similarity index 100% rename from src/cleaner/storageProviders/deleteFailureSummary.ts rename to src/cleaner/storageProviders/failuresHandling.ts diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 2f3ece4..b7002d1 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -3,11 +3,16 @@ import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; -import type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders'; +import { + mergeFailures, + type DeleteFailure, + type DeleteResourcesResult, + type IStorageProvider, + type StorageProvider, +} from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; -import { mergeFailures } from './deleteFailureSummary'; type FSStorageProviderType = Extract; diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 622d903..070569e 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,4 @@ -export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; +export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './failuresHandling'; export { FsStorageProvider, type FsConfig } from './fsStorageProvider'; export type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; From 50ba9bf1f22ab9afc99cc1dbc89815dfcba743db Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:54:39 +0300 Subject: [PATCH 31/66] refactor: add and improve log messages --- .../storageProviders/fsStorageProvider.ts | 14 ++++++-- .../storageProviders/s3StorageProvider.ts | 32 +++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index b7002d1..1b91012 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -35,10 +35,11 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); - this.logger.debug(`Using ${this.basePath} as base path for FS`); + this.logger.debug({ msg: 'Using FS storage provider', basePath: this.basePath }); } public async targetExists(storageTarget: string, relativePath: string): Promise { + this.logger.debug({ msg: `Checking if target resource exists`, basePath: storageTarget, path: relativePath }); try { await stat(join(storageTarget, relativePath)); return true; @@ -49,7 +50,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } public async delete(paths: string[], storageTarget: string): Promise { - this.logger.info({ msg: 'Deleting files from filesystem', basePath: storageTarget, count: paths.length }); + this.logger.debug({ msg: 'Deleting files from filesystem', basePath: storageTarget, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { @@ -81,6 +82,8 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, }: Extract): Promise { + this.logger.debug({ msg: `Starting FS files/dirs deletion`, pathsCount: paths.length }); + // Prevent path traversal (i.e. accessing folders above root folder) if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); @@ -112,6 +115,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private canDeleteFromFolder(path: string): void { try { accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); + this.logger.debug({ msg: `Able to delete from directory`, path }); } catch (err) { if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { throw new ConfigurationError(`FS path does not exist: ${path}`); @@ -134,10 +138,13 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } private checkPathTraversal(paths: string[]): boolean { - return paths.every((path) => { + this.logger.debug({ msg: 'Checking path traversal', paths }); + const result = paths.every((path) => { const absolutePath = resolveAbsolutePath(join(this.basePath, path)); return absolutePath.startsWith(normalizeFolderPath(this.basePath)); }); + this.logger.debug({ msg: `Path traversal check ${result ? 'succeeded' : 'failed'}` }); + return result; } // Attempts to remove any directories that became empty after file deletion. @@ -153,6 +160,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { // levelIdx 2 → { storageTarget/layer/v1 } // levelIdx 3 → { storageTarget/layer } (root ancestor, tried last) private async cleanupEmptyDirs(relativePaths: string[], storageTarget: string): Promise { + this.logger.debug({ msg: `Deleting empty directories`, pathsCount: relativePaths.length }); // Map from levelIdx → unique absolute dir paths at that depth. // Using a Set per level deduplicates dirs shared by multiple deleted files // (e.g. a shared parent directory when multiple files within it are deleted at once). diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 524fae1..dafb647 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -45,6 +45,7 @@ export interface S3Config { export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; private readonly s3Config: S3Config; + private readonly batchSize: number; public constructor( config: ConfigType, @@ -52,6 +53,7 @@ export class S3StorageProvider implements IStorageProvider { - this.logger.debug({ msg: 'Deleting objects from S3', bucket, count: paths.length }); + this.logger.debug({ msg: 'Deleting objects from S3', bucket, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); for (const chunk of getChunk(paths, Math.min(this.s3Config.delete.batchSize, S3_DELETE_OBJECTS_MAX_KEYS))) { @@ -80,7 +83,7 @@ export class S3StorageProvider implements IStorageProvider): Promise { - this.logger.debug({ msg: `Starting S3 deletion`, bucket, paths }); + this.logger.debug({ msg: `Starting S3 resources deletion`, bucket, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); if (paths.some((path) => path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion @@ -99,6 +102,7 @@ export class S3StorageProvider implements IStorageProvider { + this.logger.debug({ msg: `Checking if target resource exists`, bucket, path: relativePath }); const prefix = normalizeFolderPath(relativePath); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); @@ -137,15 +141,17 @@ export class S3StorageProvider implements IStorageProvider> => error.Key !== undefined) // Only include entries with a Key .forEach((error) => { const reason = error.Code ?? error.Message ?? 'Unknown'; const failure = failures.get(reason); + totalFailuresCount = (failure?.count ?? 0) + 1; failures.set(reason, { count: (failure?.count ?? 0) + 1, sample: error.Key }); }); - if (failures.size > 0) this.logger.warn({ msg: `Failed to delete ${failures.size} out of ${paths.length} objects` }); + if (failures.size > 0) this.logger.warn({ msg: `Failed to delete some objects`, uniqueFailureTypesCount: failures.size, totalFailuresCount }); return failures; } catch (err) { const reason = describeError(err); @@ -155,6 +161,7 @@ export class S3StorageProvider implements IStorageProvider { + this.logger.debug({ msg: 'Deleting a resource', bucket, path }); let failures: DeleteFailure = new Map(); let totalDeletedObjectsCount = 0, totalFailedObjectsCount = 0; @@ -162,16 +169,16 @@ export class S3StorageProvider implements IStorageProvider obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); @@ -220,9 +227,11 @@ export class S3StorageProvider implements IStorageProvider { try { + this.logger.debug({ msg: 'Starting iterating over matching objects', bucket, prefix, pageSize }); // First, if object exists it is removed. This is to mitigate an issue in MinIO that shadows paths sharing common path with an object. // Second, objects having this path are iterated and removed if (prefix !== undefined && (await this.resourceExists({ bucket, path: prefix }))) { + this.logger.debug({ msg: 'Matched an exact object', bucket, prefix }); yield [{ Key: prefix }]; } @@ -236,8 +245,17 @@ export class S3StorageProvider implements IStorageProvider Date: Thu, 23 Jul 2026 17:58:50 +0300 Subject: [PATCH 32/66] refactor: rename vars for clarity --- .../storageProviders/fsStorageProvider.ts | 16 ++++++++-------- .../storageProviders/s3StorageProvider.ts | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 1b91012..89ac7f3 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -38,10 +38,10 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { this.logger.debug({ msg: 'Using FS storage provider', basePath: this.basePath }); } - public async targetExists(storageTarget: string, relativePath: string): Promise { - this.logger.debug({ msg: `Checking if target resource exists`, basePath: storageTarget, path: relativePath }); + public async targetExists(basePath: string, relativePath: string): Promise { + this.logger.debug({ msg: `Checking if target resource exists`, basePath, path: relativePath }); try { - await stat(join(storageTarget, relativePath)); + await stat(join(basePath, relativePath)); return true; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; @@ -49,14 +49,14 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } } - public async delete(paths: string[], storageTarget: string): Promise { - this.logger.debug({ msg: 'Deleting files from filesystem', basePath: storageTarget, pathsCount: paths.length }); + public async delete(paths: string[], basePath: string): Promise { + this.logger.debug({ msg: 'Deleting files from filesystem', basePath, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( relativePaths.map(async (relativePath) => { - await unlink(join(storageTarget, relativePath)); + await unlink(join(basePath, relativePath)); }) ); @@ -66,7 +66,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const relativePath = relativePaths[idx]!; const error: unknown = result.reason; const reason = describeError(error); - this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); + this.logger.debug({ msg: 'Failed to delete file', path: join(basePath, relativePath), reason, error }); const chunkFailure = chunkFailures.get(reason); chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePaths[idx]! }); } @@ -74,7 +74,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { failures = mergeFailures({ source: chunkFailures, target: failures }); } - await this.cleanupEmptyDirs(paths, storageTarget); + await this.cleanupEmptyDirs(paths, basePath); return { failures }; } diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index dafb647..e61f219 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -101,9 +101,9 @@ export class S3StorageProvider implements IStorageProvider { - this.logger.debug({ msg: `Checking if target resource exists`, bucket, path: relativePath }); - const prefix = normalizeFolderPath(relativePath); + public async targetExists(bucket: string, path: string): Promise { + this.logger.debug({ msg: `Checking if target resource exists`, bucket, path }); + const prefix = normalizeFolderPath(path); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); return (result.KeyCount ?? 0) > 0; From c7f580ae0616e28c5d5f79e377d5190ce158dffa Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:43:42 +0300 Subject: [PATCH 33/66] refactor: remove unused config vars of sample size --- config/custom-environment-variables.json | 10 ---------- config/default.json | 6 +----- helm/templates/configmap.yaml | 4 ---- helm/values.yaml | 3 --- src/cleaner/strategies/tilesDeletionStrategy.ts | 2 -- tests/helpers/mocks.ts | 7 +------ 6 files changed, 2 insertions(+), 30 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 86c410e..c7c9649 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -118,16 +118,6 @@ "concurrency": { "__name": "TILES_DELETION_CONCURRENCY", "__format": "number" - }, - "failureSampleSize": { - "__name": "TILES_DELETION_FAILURE_SAMPLE_SIZE", - "__format": "number" - } - }, - "storedResourcesDeletion": { - "failureSampleSize": { - "__name": "STORED_RESOURCES_DELETION_FAILURE_SAMPLE_SIZE", - "__format": "number" } } } diff --git a/config/default.json b/config/default.json index 2dd1972..b902a1f 100644 --- a/config/default.json +++ b/config/default.json @@ -104,11 +104,7 @@ "tilesDeletion": { "s3Bucket": "", "batchSize": 1000, - "concurrency": 10, - "failureSampleSize": 3 - }, - "storedResourcesDeletion": { - "failureSampleSize": 3 + "concurrency": 10 } } } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 1860c39..4176fe6 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -71,9 +71,5 @@ data: {{- with .Values.env.strategies.tilesDeletion }} TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }} TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }} - TILES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }} - {{- end }} - {{- with .Values.env.strategies.storedResourcesDeletion }} - STORED_RESOURCES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }} {{- end }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 2203dec..e225e2d 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -134,9 +134,6 @@ env: tilesDeletion: batchSize: 1000 concurrency: 10 - failureSampleSize: 3 - storedResourcesDeletion: - failureSampleSize: 3 resources: enabled: true diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index 147df18..38f1fb1 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -25,7 +25,6 @@ const NOT_FOUND_REASONS = new Set([NoSuchKey.name, 'ENOENT']); export class TilesDeletionStrategy implements ITaskStrategy { private readonly batchSize: number; private readonly concurrency: number; - private readonly failureSampleSize: number; private readonly s3Bucket: string; private readonly fsBasePath: string; @@ -38,7 +37,6 @@ export class TilesDeletionStrategy implements ITaskStrategy ) { this.batchSize = config.get('strategies.tilesDeletion.batchSize') as unknown as number; this.concurrency = config.get('strategies.tilesDeletion.concurrency') as unknown as number; - this.failureSampleSize = config.get('strategies.tilesDeletion.failureSampleSize') as unknown as number; this.s3Bucket = config.get('strategies.tilesDeletion.s3Bucket') as unknown as string; this.fsBasePath = config.get('storage.fs.basePath') as unknown as FsConfig['basePath']; } diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 13d31c7..b960093 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -81,7 +81,6 @@ export function createMockStorageProvider = {} const values: Record = { 'strategies.tilesDeletion.batchSize': TILES_DELETION_CONFIG_DEFAULTS.batchSize, 'strategies.tilesDeletion.concurrency': TILES_DELETION_CONFIG_DEFAULTS.concurrency, - 'strategies.tilesDeletion.failureSampleSize': TILES_DELETION_CONFIG_DEFAULTS.failureSampleSize, 'strategies.tilesDeletion.s3Bucket': TILES_DELETION_CONFIG_DEFAULTS.s3Bucket, 'storage.fs.basePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, ...overrides, @@ -100,13 +98,10 @@ export function createMockStrategyConfig(overrides: Record = {} // ─── Strategy Config (DeleteStoredResourcesStrategy) ──────────────────────────── -export const STORED_RESOURCES_DELETION_CONFIG_DEFAULTS = { - failureSampleSize: 3, -} as const; +export const STORED_RESOURCES_DELETION_CONFIG_DEFAULTS = {} as const; export function createMockStoredResourcesDeletionStrategyConfig(overrides: Record = {}): ConfigType { const values: Record = { - 'strategies.storedResourcesDeletion.failureSampleSize': STORED_RESOURCES_DELETION_CONFIG_DEFAULTS.failureSampleSize, ...overrides, }; return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; From 04d5fdd0226eaa1c7574b096709421c6245967fa Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:56:24 +0300 Subject: [PATCH 34/66] refactor: fix and improve logs --- .../storageProviders/s3StorageProvider.ts | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index e61f219..3f0a690 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -151,7 +151,13 @@ export class S3StorageProvider implements IStorageProvider 0) this.logger.warn({ msg: `Failed to delete some objects`, uniqueFailureTypesCount: failures.size, totalFailuresCount }); + if (failures.size > 0) + this.logger.warn({ + msg: `Failed to delete some objects`, + totalFailuresCount, + uniqueFailureTypesCount: failures.size, + failureTypes: Array.from(failures.keys()), + }); return failures; } catch (err) { const reason = describeError(err); @@ -178,6 +184,7 @@ export class S3StorageProvider implements IStorageProvider obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); @@ -189,25 +196,25 @@ export class S3StorageProvider implements IStorageProvider (failedObjectsCount += chunkFailure.count)); const deletedObjectsCount = keys.length - failedObjectsCount; totalDeletedObjectsCount += deletedObjectsCount; totalFailedObjectsCount += failedObjectsCount; this.logger.debug({ - msg: `Successfully deleted ${deletedObjectsCount} objects. Totally ${totalDeletedObjectsCount} successfully deleted objects`, + msg: 'Completed processing current page of objects', + deletedObjectsCount, + totalDeletedObjectsCount, + failedObjectsCount, + totalFailedObjectsCount, }); - - if (failedObjectsCount > 0) - this.logger.debug({ - msg: `Could not delete ${failedObjectsCount} objects. Totally ${totalFailedObjectsCount} objects could not be deleted`, - }); } - this.logger.debug({ msg: 'Deletion completed', path, totalDeletedObjectsCount, totalFailedObjectsCount }); + this.logger.debug({ msg: 'Resource deletion completed', path, totalDeletedObjectsCount, totalFailedObjectsCount }); return failures; } catch (err) { this.logger.error({ - msg: 'Stream of objects was interrupted by an error', + msg: 'Stream of objects for deletion was interrupted by an error', path, totalDeletedObjectsCount, totalFailedObjectsCount, @@ -227,7 +234,7 @@ export class S3StorageProvider implements IStorageProvider { try { - this.logger.debug({ msg: 'Starting iterating over matching objects', bucket, prefix, pageSize }); + this.logger.debug({ msg: 'Starting iteration over matching objects', bucket, prefix, pageSize }); // First, if object exists it is removed. This is to mitigate an issue in MinIO that shadows paths sharing common path with an object. // Second, objects having this path are iterated and removed if (prefix !== undefined && (await this.resourceExists({ bucket, path: prefix }))) { From fa5b8f2d2410d51efeb914aa5799421416b8b288 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:00:55 +0300 Subject: [PATCH 35/66] fix: map data structure handling --- src/cleaner/storageProviders/failuresHandling.ts | 2 +- src/cleaner/strategies/deleteStoredResourcesStrategy.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cleaner/storageProviders/failuresHandling.ts b/src/cleaner/storageProviders/failuresHandling.ts index 0f40c1f..e9f0b07 100644 --- a/src/cleaner/storageProviders/failuresHandling.ts +++ b/src/cleaner/storageProviders/failuresHandling.ts @@ -34,7 +34,7 @@ export function summarizeDeleteFailures({ failures }: DeleteResourcesResult): De failuresCount += count; } - const summary = Object.entries(failures) + const summary = Array.from(failures.entries()) .sort(([, { count: a }], [, { count: b }]) => b - a) .map(([reason, { count }]) => `${reason}=${count}`) .join(', '); diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts index fc66b20..94e2c36 100644 --- a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -35,14 +35,14 @@ export class DeleteStoredResourcesStrategy implements ITaskStrategy 0) { + if (failures.size > 0) { const { failuresCount, summary } = summarizeDeleteFailures({ failures }); this.logger.error({ msg: 'Deletion failed', provider: params.storageProvider, paths, failuresCount, - failures, + summary, }); throw new RecoverableError(`Failed to delete ${failuresCount} objects. Reasons: ${summary}.`); } From 5657069342d226a9bdb3cca41342abd96501afd9 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:53:19 +0300 Subject: [PATCH 36/66] style: string formatting --- src/cleaner/storageProviders/fsStorageProvider.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 89ac7f3..002451c 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -39,7 +39,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } public async targetExists(basePath: string, relativePath: string): Promise { - this.logger.debug({ msg: `Checking if target resource exists`, basePath, path: relativePath }); + this.logger.debug({ msg: 'Checking if target resource exists', basePath, path: relativePath }); try { await stat(join(basePath, relativePath)); return true; @@ -82,10 +82,10 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, }: Extract): Promise { - this.logger.debug({ msg: `Starting FS files/dirs deletion`, pathsCount: paths.length }); + this.logger.debug({ msg: 'Starting FS files/dirs deletion', pathsCount: paths.length }); // Prevent path traversal (i.e. accessing folders above root folder) - if (!this.checkPathTraversal(paths)) throw new UnrecoverableError(`Cannot delete files/folders outside base path or base path itself`); + if (!this.checkPathTraversal(paths)) throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself'); let failures: DeleteFailure = new Map(); @@ -115,7 +115,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private canDeleteFromFolder(path: string): void { try { accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); - this.logger.debug({ msg: `Able to delete from directory`, path }); + this.logger.debug({ msg: 'Able to delete from directory', path }); } catch (err) { if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { throw new ConfigurationError(`FS path does not exist: ${path}`); @@ -160,7 +160,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { // levelIdx 2 → { storageTarget/layer/v1 } // levelIdx 3 → { storageTarget/layer } (root ancestor, tried last) private async cleanupEmptyDirs(relativePaths: string[], storageTarget: string): Promise { - this.logger.debug({ msg: `Deleting empty directories`, pathsCount: relativePaths.length }); + this.logger.debug({ msg: 'Deleting empty directories', pathsCount: relativePaths.length }); // Map from levelIdx → unique absolute dir paths at that depth. // Using a Set per level deduplicates dirs shared by multiple deleted files // (e.g. a shared parent directory when multiple files within it are deleted at once). From 238dd4ffb3ed02d8535b40d20b67f7e3e1f61f23 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:50 +0300 Subject: [PATCH 37/66] refactor: improve logs --- .../storageProviders/s3StorageProvider.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 3f0a690..c875949 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -102,7 +102,7 @@ export class S3StorageProvider implements IStorageProvider { - this.logger.debug({ msg: `Checking if target resource exists`, bucket, path }); + this.logger.debug({ msg: 'Checking if target resource exists', bucket, path }); const prefix = normalizeFolderPath(path); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); @@ -116,14 +116,14 @@ export class S3StorageProvider implements IStorageProvider { try { - this.logger.debug({ msg: `Checking bucket exists`, bucket }); + this.logger.debug({ msg: 'Checking bucket exists', bucket }); const command = new HeadBucketCommand({ Bucket: bucket }); await this.s3Client.send(command); // If it resolves, the bucket exists and you have permission to access it - this.logger.debug({ msg: `Bucket exists`, bucket }); + this.logger.debug({ msg: 'Bucket exists', bucket }); return true; } catch (err) { if (err instanceof NotFound) { - this.logger.error({ msg: `Bucket does not exist`, bucket, err }); + this.logger.error({ msg: 'Bucket does not exist', bucket, err }); return false; } const reason = describeError(err); @@ -153,7 +153,7 @@ export class S3StorageProvider implements IStorageProvider 0) this.logger.warn({ - msg: `Failed to delete some objects`, + msg: 'Failed to delete some objects', totalFailuresCount, uniqueFailureTypesCount: failures.size, failureTypes: Array.from(failures.keys()), @@ -267,9 +267,9 @@ export class S3StorageProvider implements IStorageProvider Date: Sun, 26 Jul 2026 17:57:21 +0300 Subject: [PATCH 38/66] refactor: pr comment use storage type --- src/cleaner/storageProviders/iStorageProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index 99c5fe5..509ece1 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -1,4 +1,4 @@ -import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import type { DeleteStoredResourcesParams, Storage } from '@map-colonies/raster-shared'; /** * A storage for failures with additional metadata. @@ -9,7 +9,7 @@ export interface DeleteResourcesResult { failures: DeleteFailure; } -export type StorageProvider = DeleteStoredResourcesParams['storageProvider']; +export type StorageProvider = Storage['storageProvider']; export interface IStorageProvider { /** From 0bb370c6200e4041307e9741f000481a7fd718d1 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:55:04 +0300 Subject: [PATCH 39/66] fix: report full path failure --- src/cleaner/storageProviders/fsStorageProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 002451c..5a7d3fe 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -103,7 +103,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const reason = describeError(result.reason); this.logger.error({ msg: 'Failed to delete layer directory', fullPath, reason, err: result.reason }); const chunkFailure = chunkFailures.get(reason); - chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePaths[idx]! }); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? fullPath }); } } failures = mergeFailures({ source: chunkFailures, target: failures }); From 2c8517d106e3b3381366d1aa99bd9f4df72dbaad Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:11:43 +0300 Subject: [PATCH 40/66] fix: keep existing example if exists --- src/cleaner/storageProviders/s3StorageProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index c875949..68d85d9 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -148,7 +148,7 @@ export class S3StorageProvider implements IStorageProvider 0) From 31d9f956b3a4d077d99a8c0f2dce2b8052054a94 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:28:30 +0300 Subject: [PATCH 41/66] build: update @map-colonies/raster-shared --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 915d6fd..958df41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.3.0-alpha.1", + "@map-colonies/raster-shared": "^8.3.0-alpha.2", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", @@ -2543,9 +2543,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "8.3.0-alpha.1", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.1.tgz", - "integrity": "sha512-KP/n3ZPcG7ULHrtX3h9fMOTNnF5Qkos1N9ZPjawi/5yiDHit9S5V9sFI4Tgv5aMchgSS0zXu5gL0ybA4/CliYQ==", + "version": "8.3.0-alpha.2", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.2.tgz", + "integrity": "sha512-P2p0MddLonzxsG/MSkcKopfI05JsaWxe9BpfYPrVllZ7wCbrzyBUnsYsfP3Rl6jHeNlqWWPCJdHkpzvgTi0FOQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index d15fb70..ecb938c 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.3.0-alpha.1", + "@map-colonies/raster-shared": "^8.3.0-alpha.2", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", From b1b6036cc586ffae05020782cfb696c22e9cc2c5 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:38:44 +0300 Subject: [PATCH 42/66] refactor: restore previous behavior of sample logging --- src/cleaner/storageProviders/failuresHandling.ts | 11 ++++++----- .../strategies/deleteStoredResourcesStrategy.ts | 5 +++-- src/cleaner/strategies/tilesDeletionStrategy.ts | 9 +++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/cleaner/storageProviders/failuresHandling.ts b/src/cleaner/storageProviders/failuresHandling.ts index e9f0b07..b8ccc9f 100644 --- a/src/cleaner/storageProviders/failuresHandling.ts +++ b/src/cleaner/storageProviders/failuresHandling.ts @@ -9,6 +9,8 @@ export interface DeleteFailureSummary { failuresCount: number; /** Reasons formatted descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ summary: string; + /** Samples of failing resources preserving input order. */ + samples: string[]; } export const mergeFailures = ({ source, target }: { source: DeleteFailure; target: DeleteFailure }): DeleteFailure => { @@ -34,9 +36,8 @@ export function summarizeDeleteFailures({ failures }: DeleteResourcesResult): De failuresCount += count; } - const summary = Array.from(failures.entries()) - .sort(([, { count: a }], [, { count: b }]) => b - a) - .map(([reason, { count }]) => `${reason}=${count}`) - .join(', '); - return { failuresCount, summary }; + const sortedFailures = Array.from(failures.entries()).sort(([, { count: a }], [, { count: b }]) => b - a); + const summary = sortedFailures.map(([reason, { count }]) => `${reason}=${count}`).join(', '); + const samples = sortedFailures.map(([reason, { sample }]) => `${sample} (${reason})`); + return { failuresCount, summary, samples }; } diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts index 94e2c36..36441f3 100644 --- a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -36,15 +36,16 @@ export class DeleteStoredResourcesStrategy implements ITaskStrategy 0) { - const { failuresCount, summary } = summarizeDeleteFailures({ failures }); + const { failuresCount, samples, summary } = summarizeDeleteFailures({ failures }); this.logger.error({ msg: 'Deletion failed', provider: params.storageProvider, paths, failuresCount, summary, + samples, }); - throw new RecoverableError(`Failed to delete ${failuresCount} objects. Reasons: ${summary}.`); + throw new RecoverableError(`Failed to delete ${failuresCount} objects. Reasons: ${summary}. Samples: ${samples.join(', ')}`); } this.logger.info({ diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index 38f1fb1..124de25 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -84,17 +84,18 @@ export class TilesDeletionStrategy implements ITaskStrategy const deletedCount = totalTiles - retryable.size - notFound.size; if (retryable.size > 0) { - const { failuresCount, summary } = summarizeDeleteFailures({ failures: retryable }); + const { failuresCount, samples, summary } = summarizeDeleteFailures({ failures: retryable }); this.logger.error({ msg: 'Tiles deletion partially failed', totalTiles, - failedCount: retryable.size, + uniqueFailureTypesCount: retryable.size, notFoundCount: notFound.size, deletedCount, - failuresCount, + totalFailuresCount: failuresCount, summary, + samples, }); - throw new RecoverableError(`Failed to delete ${retryable.size} tiles. Reasons: ${summary}.`); + throw new RecoverableError(`Failed to delete ${failuresCount} tiles. Reasons: ${summary}. Samples: ${samples.join(', ')}`); } if (notFound.size > 0) { From 7610350843244ba77a6c289f1340f81458772c4f Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:39:20 +0300 Subject: [PATCH 43/66] fix: error count --- src/cleaner/strategies/tilesDeletionStrategy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index 124de25..002c473 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -192,7 +192,7 @@ export class TilesDeletionStrategy implements ITaskStrategy this.logger.error({ msg: 'Batch delete threw unexpectedly', reason, error }); const batch = pendingBatches[index] ?? []; const failure = failures.get(reason); - failures.set(reason, { count: (failure?.count ?? 0) + 1, sample: failure?.sample ?? batch[0]! }); + failures.set(reason, { count: (failure?.count ?? 0) + batch.length, sample: failure?.sample ?? batch[0]! }); } } pendingBatches.length = 0; From 63af7045fcd299bc4bee1802ab27b7febd3d4b9b Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:41:18 +0300 Subject: [PATCH 44/66] fix: total errors count --- src/cleaner/storageProviders/s3StorageProvider.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 68d85d9..95439f3 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -147,8 +147,9 @@ export class S3StorageProvider implements IStorageProvider { const reason = error.Code ?? error.Message ?? 'Unknown'; const failure = failures.get(reason); - totalFailuresCount = (failure?.count ?? 0) + 1; - failures.set(reason, { count: (failure?.count ?? 0) + 1, sample: failure?.sample ?? error.Key }); + const failuresCount = (failure?.count ?? 0) + 1; + totalFailuresCount += failuresCount; + failures.set(reason, { count: failuresCount, sample: failure?.sample ?? error.Key }); }); if (failures.size > 0) From 2b9d787259d8d16892b90b68a69124adf24e2381 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:48:02 +0300 Subject: [PATCH 45/66] docs: update interface properties description --- src/cleaner/storageProviders/failuresHandling.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cleaner/storageProviders/failuresHandling.ts b/src/cleaner/storageProviders/failuresHandling.ts index b8ccc9f..6c6c4ed 100644 --- a/src/cleaner/storageProviders/failuresHandling.ts +++ b/src/cleaner/storageProviders/failuresHandling.ts @@ -7,9 +7,9 @@ import type { DeleteFailure, DeleteResourcesResult } from './iStorageProvider'; export interface DeleteFailureSummary { /** Count of all failures */ failuresCount: number; - /** Reasons formatted descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ + /** Reasons formatted ordered by descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ summary: string; - /** Samples of failing resources preserving input order. */ + /** Samples of failing resources ordered by descending by count. */ samples: string[]; } From 966b8392d2091450a1a9cc960b24b8c2c1bc3e7b Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:00:47 +0300 Subject: [PATCH 46/66] test: rename test file and update tests --- .../deleteFailureSummary.spec.ts | 77 ------------------- .../storageProviders/failuresHandling.spec.ts | 51 ++++++++++++ 2 files changed, 51 insertions(+), 77 deletions(-) delete mode 100644 tests/storageProviders/deleteFailureSummary.spec.ts create mode 100644 tests/storageProviders/failuresHandling.spec.ts diff --git a/tests/storageProviders/deleteFailureSummary.spec.ts b/tests/storageProviders/deleteFailureSummary.spec.ts deleted file mode 100644 index 6f669cc..0000000 --- a/tests/storageProviders/deleteFailureSummary.spec.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { describe, it, expect } from 'vitest'; -import { summarizeDeleteFailures } from '@src/cleaner/storageProviders'; -import type { DeleteFailure } from '@src/cleaner/storageProviders'; - -describe('summarizeDeleteFailures', () => { - it('should return zero counts, empty summary and empty sample for an empty input', () => { - const result = summarizeDeleteFailures([], 5); - expect(result).toEqual({ counts: {}, summary: '', sample: [] }); - }); - - it('should count occurrences per reason', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'ENOENT' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'EACCES' }, - ]; - - const { counts } = summarizeDeleteFailures(failures, 3); - - expect(counts).toEqual({ ENOENT: 2, EACCES: 1 }); - }); - - it('should format the summary with reasons sorted by descending count', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'EACCES' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'ENOENT' }, - { path: 'd', reason: 'ENOENT' }, - ]; - - const { summary } = summarizeDeleteFailures(failures, 3); - - expect(summary).toBe('ENOENT=3, EACCES=1'); - }); - - it('should annotate each sampled path with its reason', () => { - const failures: DeleteFailure[] = [ - { path: 'tile/1.png', reason: 'ENOENT' }, - { path: 'tile/2.png', reason: 'EACCES' }, - ]; - - const { sample } = summarizeDeleteFailures(failures, 3); - - expect(sample).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); - }); - - it('should cap the sample to sampleSize and preserve input order', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'ENOENT' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'ENOENT' }, - { path: 'd', reason: 'ENOENT' }, - ]; - - const { sample } = summarizeDeleteFailures(failures, 2); - - expect(sample).toEqual(['a (ENOENT)', 'b (ENOENT)']); - expect(sample).length(2); - }); - - it('should return all failures in the sample when sampleSize exceeds input length', () => { - const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; - - const { sample } = summarizeDeleteFailures(failures, 10); - - expect(sample).toEqual(['a (ENOENT)']); - }); - - it('should return an empty sample when sampleSize is 0', () => { - const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; - - const { sample } = summarizeDeleteFailures(failures, 0); - - expect(sample).toEqual([]); - }); -}); diff --git a/tests/storageProviders/failuresHandling.spec.ts b/tests/storageProviders/failuresHandling.spec.ts new file mode 100644 index 0000000..d1863ce --- /dev/null +++ b/tests/storageProviders/failuresHandling.spec.ts @@ -0,0 +1,51 @@ +import { describe, it, expect } from 'vitest'; +import { summarizeDeleteFailures } from '@src/cleaner/storageProviders'; + +describe('#summarizeDeleteFailures', () => { + it('should return zero counts, empty summary for an empty input', () => { + const failures = { failures: new Map() }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toEqual({ failuresCount: 0, summary: '', samples: [] }); + }); + + it('should format the summary with reasons and samples sorted by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 3, sample: 'b' }], + ]), + }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toStrictEqual({ failuresCount: 4, summary: 'ENOENT=3, EACCES=1', samples: ['b (ENOENT)', 'a (EACCES)'] }); + }); + + it('should annotate each sampled path with its reason', () => { + const failures = { + failures: new Map([ + ['ENOENT', { count: 1, sample: 'tile/1.png' }], + ['EACCES', { count: 1, sample: 'tile/2.png' }], + ]), + }; + + const { samples } = summarizeDeleteFailures(failures); + + expect(samples).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); + }); + + it('should order samples by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 4, sample: 'b' }], + ]), + }; + + const { samples } = summarizeDeleteFailures(failures); + + expect(samples).toEqual(['b (ENOENT)', 'a (EACCES)']); + }); +}); From a48b335c40b951ae1f589f48629bf9bb37fda5fc Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:35:15 +0300 Subject: [PATCH 47/66] feat: consume subpath for resource deletion --- src/cleaner/storageProviders/fsStorageProvider.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 5a7d3fe..3b61b28 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -81,17 +81,20 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, + subPath, }: Extract): Promise { - this.logger.debug({ msg: 'Starting FS files/dirs deletion', pathsCount: paths.length }); + this.logger.debug({ msg: 'Starting FS files/dirs deletion', subPath, pathsCount: paths.length }); + + const relativePaths = paths.map((path) => join(subPath, path)); // Prevent path traversal (i.e. accessing folders above root folder) - if (!this.checkPathTraversal(paths)) throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself'); + if (!this.checkPathTraversal(relativePaths)) throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself'); let failures: DeleteFailure = new Map(); - for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { + for (const relativePathsChunk of getChunk(relativePaths, this.fsConfig.delete.batchSize)) { const results = await Promise.allSettled( - relativePaths.map(async (relativePath) => { + relativePathsChunk.map(async (relativePath) => { await rm(join(this.basePath, relativePath), { recursive: true, force: true }); }) ); @@ -99,7 +102,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const chunkFailures: DeleteFailure = new Map(); for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { - const fullPath = join(this.basePath, relativePaths[idx]!); + const fullPath = join(this.basePath, relativePathsChunk[idx]!); const reason = describeError(result.reason); this.logger.error({ msg: 'Failed to delete layer directory', fullPath, reason, err: result.reason }); const chunkFailure = chunkFailures.get(reason); From 0aafe585403fae75c9486931c70aef5aadcc0f36 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:30:13 +0300 Subject: [PATCH 48/66] chore: update FS_BASE_PATH to use internalPvc.mountPath only --- helm/templates/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 4176fe6..0a115c8 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -4,7 +4,7 @@ {{- $s3 := ($storage.s3) | default dict -}} {{- $fs := ($storage.fs) | default dict -}} {{- $internalPvc := (($fs).internalPvc) | default dict -}} -{{- $tilesFSBasePath := clean (printf "/%s/%s" $internalPvc.mountPath ($internalPvc.tilesSubPath | default "tiles")) -}} +{{- $fsBasePath := clean (printf "/%s" $internalPvc.mountPath) -}} {{- if .Values.enabled -}} apiVersion: v1 kind: ConfigMap @@ -65,7 +65,7 @@ data: {{- end }} {{- if has "FS" $storage.cleanupStorageProviders }} FS_DELETE_BATCH_SIZE: {{ $fs.delete.batchSize | default 1000 | quote }} - FS_BASE_PATH: {{ $tilesFSBasePath | quote }} + FS_BASE_PATH: {{ $fsBasePath | quote }} {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} {{- with .Values.env.strategies.tilesDeletion }} From d5c53ca40bf55e0ef63915cfbb096d28736624cf Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:09:25 +0300 Subject: [PATCH 49/66] feat: add support for available subPaths in storage configuration and update related logic --- config/custom-environment-variables.json | 6 +++++- config/default.json | 3 ++- helm/templates/configmap.yaml | 7 +++++++ src/cleaner/storageProviders/fsStorageProvider.ts | 10 ++++++++-- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index c7c9649..9c3a880 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -105,7 +105,11 @@ "__format": "number" } }, - "basePath": "FS_BASE_PATH" + "basePath": "FS_BASE_PATH", + "subPaths": { + "__name": "FS_SUB_PATHS", + "__format": "json" + } } }, "strategies": { diff --git a/config/default.json b/config/default.json index b902a1f..4d9c909 100644 --- a/config/default.json +++ b/config/default.json @@ -97,7 +97,8 @@ "delete": { "batchSize": 1000 }, - "basePath": "/tiles" + "basePath": "/tiles", + "subPaths": {} } }, "strategies": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 0a115c8..abc81c9 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -66,6 +66,13 @@ data: {{- if has "FS" $storage.cleanupStorageProviders }} FS_DELETE_BATCH_SIZE: {{ $fs.delete.batchSize | default 1000 | quote }} FS_BASE_PATH: {{ $fsBasePath | quote }} + {{- $subPaths := dict -}} + {{- range $key, $val := $internalPvc -}} + {{- if and (hasSuffix "SubPath" $key) $val -}} + {{- $_ := set $subPaths (trimSuffix "SubPath" $key) (clean $val) -}} + {{- end -}} + {{- end }} + FS_SUB_PATHS: {{ $subPaths | toJson | quote }} {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} {{- with .Values.env.strategies.tilesDeletion }} diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 3b61b28..8bc4c98 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -21,17 +21,20 @@ export interface FsConfig { batchSize: number; }; basePath: string; + subPaths: Record; } export class FsStorageProvider implements IStorageProvider<'FS'> { private readonly fsConfig: FsConfig; private readonly basePath: string; + private readonly subPaths: string[]; public constructor( private readonly config: ConfigType, private readonly logger: Logger ) { this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; + this.subPaths = Object.values(this.fsConfig.subPaths); if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); @@ -88,7 +91,8 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const relativePaths = paths.map((path) => join(subPath, path)); // Prevent path traversal (i.e. accessing folders above root folder) - if (!this.checkPathTraversal(relativePaths)) throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself'); + if (!this.checkPathTraversal(relativePaths)) + throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself and must match a valid configured path'); let failures: DeleteFailure = new Map(); @@ -143,8 +147,10 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private checkPathTraversal(paths: string[]): boolean { this.logger.debug({ msg: 'Checking path traversal', paths }); const result = paths.every((path) => { + const startsWithAllowedSubPath = this.subPaths.some((subPath) => path.startsWith(subPath)); const absolutePath = resolveAbsolutePath(join(this.basePath, path)); - return absolutePath.startsWith(normalizeFolderPath(this.basePath)); + const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.basePath)); + return startsWithAllowedSubPath && startsWithBasePath; }); this.logger.debug({ msg: `Path traversal check ${result ? 'succeeded' : 'failed'}` }); return result; From 231afa15e6332dee33f1eb1d1e5ef4b86810033d Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:33:24 +0300 Subject: [PATCH 50/66] refactor: modify log messages --- src/cleaner/storageProviders/fsStorageProvider.ts | 2 +- src/cleaner/storageProviders/s3StorageProvider.ts | 2 +- src/cleaner/strategies/deleteStoredResourcesStrategy.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 8bc4c98..c3963a3 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -38,7 +38,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); - this.logger.debug({ msg: 'Using FS storage provider', basePath: this.basePath }); + this.logger.debug({ msg: 'Loaded FS storage provider', basePath: this.basePath }); } public async targetExists(basePath: string, relativePath: string): Promise { diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 95439f3..8e2e773 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -64,7 +64,7 @@ export class S3StorageProvider implements IStorageProvider { diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts index 36441f3..157b371 100644 --- a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -27,10 +27,11 @@ export class DeleteStoredResourcesStrategy implements ITaskStrategy Date: Wed, 29 Jul 2026 11:09:11 +0300 Subject: [PATCH 51/66] test: update tests with changes and add some additional tests --- tests/helpers/mocks.ts | 7 +- .../fsStorageProvider.spec.ts | 82 +++++++++++++------ .../s3StorageProvider.spec.ts | 69 ++++++++-------- .../deleteStoredResourcesStrategy.spec.ts | 10 ++- .../strategies/tilesDeletionStrategy.spec.ts | 45 +++++----- 5 files changed, 121 insertions(+), 92 deletions(-) diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index b960093..300ea03 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -70,8 +70,8 @@ export function createMockErrorHandler(defaultDecision: ErrorDecision = { should export function createMockStorageProvider(): IStorageProvider { return { - delete: vi.fn().mockResolvedValue([]), - deleteResources: vi.fn().mockResolvedValue({ failures: [] }), + delete: vi.fn().mockResolvedValue({ failures: new Map() }), + deleteResources: vi.fn().mockResolvedValue({ failures: new Map() }), targetExists: vi.fn().mockResolvedValue(true), }; } @@ -134,6 +134,9 @@ export const FS_STORAGE_CONFIG_DEFAULTS = { batchSize: 3, }, basePath: '/test/tiles', + subPaths: { + tiles: 'test/artifacts/tiles', + }, } as const satisfies FsConfig; export function createMockFsConfig(overrides: Record = {}): ConfigType { diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index ecdb749..598f026 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -71,9 +71,9 @@ describe('FsStorageProvider', () => { }); describe('#delete', () => { - it('should return empty array for empty input', async () => { + it('should return empty failures map for empty input', async () => { const result = await provider.delete([], BASE_PATH); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); expect(unlink).not.toHaveBeenCalled(); }); @@ -94,9 +94,9 @@ describe('FsStorageProvider', () => { } }); - it('should return empty array when all unlinks succeed', async () => { + it('should return empty failures map when all unlinks succeed', async () => { const result = await provider.delete(['tile/10/0/0.png', 'tile/10/0/1.png'], BASE_PATH); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should treat ENOENT as a failed deletion tagged with ENOENT reason', async () => { @@ -105,7 +105,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'ENOENT' }]); + expect(result).toEqual({ failures: new Map([['ENOENT', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should return failed path with reason for non-ENOENT errors', async () => { @@ -114,7 +114,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'EACCES' }]); + expect(result).toEqual({ failures: new Map([['EACCES', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should fall back to error message when error has no errno code', async () => { @@ -122,7 +122,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'disk on fire' }]); + expect(result).toEqual({ failures: new Map([['disk on fire', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should fall back to "Unknown" when error has neither errno code nor message', async () => { @@ -130,7 +130,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'Unknown' }]); + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should handle mixed success, ENOENT and real errors', async () => { @@ -145,10 +145,12 @@ describe('FsStorageProvider', () => { const paths = ['tile/10/0/0.png', 'tile/10/0/1.png', 'tile/10/0/2.png']; const result = await provider.delete(paths, BASE_PATH); - expect(result).toEqual([ - { path: 'tile/10/0/1.png', reason: 'ENOENT' }, - { path: 'tile/10/0/2.png', reason: 'EACCES' }, - ]); + expect(result).toEqual({ + failures: new Map([ + ['ENOENT', { count: 1, sample: 'tile/10/0/1.png' }], + ['EACCES', { count: 1, sample: 'tile/10/0/2.png' }], + ]), + }); }); it('should use relative path (not the full absolute path) in failure entries', async () => { @@ -158,8 +160,8 @@ describe('FsStorageProvider', () => { const relativePath = 'layer/v1/10/5/3.png'; const result = await provider.delete([relativePath], BASE_PATH); - expect(result).toEqual([{ path: relativePath, reason: 'EACCES' }]); - expect(result[0]!.path).not.toContain(BASE_PATH); + expect(result).toEqual({ failures: new Map([['EACCES', { count: 1, sample: relativePath }]]) }); + expect(Array.from(result.failures.values())[0]?.sample).not.toMatch(`^${BASE_PATH}*`); }); describe('cleanupEmptyDirs', () => { @@ -192,8 +194,10 @@ describe('FsStorageProvider', () => { const enotempty = Object.assign(new Error('ENOTEMPTY'), { code: 'ENOTEMPTY' }); vi.mocked(rmdir).mockRejectedValue(enotempty); + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + // Should not throw and should return correct failed paths - await expect(provider.delete(['tile/10/0/0.png'], BASE_PATH)).resolves.toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should not call rmdir when input is empty', async () => { @@ -204,49 +208,72 @@ describe('FsStorageProvider', () => { }); describe('#deleteResources', () => { + const FS_SUB_PATH = 'test/artifacts/tiles'; const RELATIVE_PATH = 'layer/v1'; it('should successfully call delete all files and return without failures', async () => { - const result = await provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); - expect(rm).toHaveBeenCalledWith(join(BASE_PATH, RELATIVE_PATH), { recursive: true, force: true }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); + }); + + it('should successfully call delete all files and return without failures for multiple paths', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH, RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); + }); + + it('should successfully call delete all files and return without failures for multiple paths', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH, `${RELATIVE_PATH}/old`], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); }); it('should throw UnrecoverableError when a path escapes the base path via traversal', async () => { - await expect(provider.deleteResources({ paths: ['../../../etc/passwd'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + const result = provider.deleteResources({ paths: ['../../../../etc/passwd'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); expect(rm).not.toHaveBeenCalled(); }); it('should throw UnrecoverableError when only one of several paths escapes the base path', async () => { - await expect(provider.deleteResources({ paths: [RELATIVE_PATH, '../../escape'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + const result = provider.deleteResources({ paths: [RELATIVE_PATH, '../../../../escape'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); expect(rm).not.toHaveBeenCalled(); }); it('should throw UnrecoverableError when a path resolves to the base path itself', async () => { - await expect(provider.deleteResources({ paths: [''], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + const result = provider.deleteResources({ paths: ['../../../'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); expect(rm).not.toHaveBeenCalled(); }); it('should throw UnrecoverableError when a path resolves to the base path itself via "."', async () => { - await expect(provider.deleteResources({ paths: ['.'], storageProvider: 'FS' })).rejects.toThrow(UnrecoverableError); + const result = provider.deleteResources({ paths: ['../../../.'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); expect(rm).not.toHaveBeenCalled(); }); it('should return failures entry when rm rejects', async () => { vi.mocked(rm).mockRejectedValue(new Error('EACCES')); - const result = await provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); expect(result).toEqual({ - failures: [{ path: join(BASE_PATH, RELATIVE_PATH), reason: 'EACCES' }], + failures: new Map([['EACCES', { count: 1, sample: join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH) }]]), }); }); it('should not throw when rm rejects', async () => { vi.mocked(rm).mockRejectedValue(new Error('Permission denied')); - const result = provider.deleteResources({ paths: [RELATIVE_PATH], storageProvider: 'FS' }); + const result = provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); await expect(result).resolves.not.toThrow(); }); @@ -291,7 +318,10 @@ describe('FsStorageProvider', () => { it('should resolve a relative base path (without a leading separator) to an absolute path', () => { const relativeConfig = { - get: vi.fn().mockReturnValue({ basePath: 'relative/tiles', delete: { batchSize: 100 } } satisfies FsConfig), + get: vi.fn().mockReturnValue({ + ...FS_STORAGE_CONFIG_DEFAULTS, + ...({ basePath: 'relative/tiles' } satisfies Pick), + }), } as unknown as ConfigType; expect(() => new FsStorageProvider(relativeConfig, createMockLogger())).not.toThrow(); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index ade8544..392cc6a 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -60,9 +60,9 @@ describe('S3StorageProvider', () => { mockSend.mockResolvedValue({ Errors: [] }); }); - it('should return empty array for empty input', async () => { + it('should return empty failures map for empty input', async () => { const result = await provider.delete([], BUCKET); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); expect(mockSend).not.toHaveBeenCalled(); }); @@ -78,12 +78,12 @@ describe('S3StorageProvider', () => { expect(mockSend).toHaveBeenCalledTimes(1); }); - it('should return empty array when all deletes succeed', async () => { + it('should return empty failures map when all deletes succeed', async () => { const paths = ['a.txt', 'b.txt']; const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should return failed paths tagged with the S3 error Code', async () => { @@ -94,7 +94,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'AccessDenied' }]); + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'a.txt' }]]) }); }); it('should treat NoSuchKey as a failed deletion tagged with NoSuchKey reason', async () => { @@ -105,7 +105,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([{ path: 'missing.txt', reason: 'NoSuchKey' }]); + expect(result).toEqual({ failures: new Map([['NoSuchKey', { count: 1, sample: 'missing.txt' }]]) }); }); it('should fall back to Message when error has no Code', async () => { @@ -115,7 +115,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt'], BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'Something bad' }]); + expect(result).toEqual({ failures: new Map([['Something bad', { count: 1, sample: 'a.txt' }]]) }); }); it('should fall back to "Unknown" when error has neither Code nor Message', async () => { @@ -125,7 +125,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt'], BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'Unknown' }]); + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'a.txt' }]]) }); }); it('should return all errors including NoSuchKey with their codes', async () => { @@ -140,12 +140,13 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt', 'b.txt', 'c.txt', 'd.txt'], BUCKET); - expect(result).toEqual([ - { path: 'a.txt', reason: 'NoSuchKey' }, - { path: 'b.txt', reason: 'AccessDenied' }, - { path: 'c.txt', reason: 'NoSuchKey' }, - { path: 'd.txt', reason: 'InternalError' }, - ]); + expect(result).toEqual({ + failures: new Map([ + ['NoSuchKey', { count: 2, sample: 'a.txt' }], + ['AccessDenied', { count: 1, sample: 'b.txt' }], + ['InternalError', { count: 1, sample: 'd.txt' }], + ]), + }); }); it('should batch paths into chunks of 1000 (S3 limit)', async () => { @@ -194,10 +195,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([ - { path: 'object-0.txt', reason: 'AccessDenied' }, - { path: 'object-1000.txt', reason: 'AccessDenied' }, - ]); + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'object-0.txt' }]]) }); }); it('should add entire chunk to failures tagged with the thrown error when send rejects', async () => { @@ -206,10 +204,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([ - { path: 'a.txt', reason: 'Network error' }, - { path: 'b.txt', reason: 'Network error' }, - ]); + expect(result).toEqual({ failures: new Map([['Network error', { count: 2, sample: 'a.txt' }]]) }); }); }); @@ -280,7 +275,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -311,7 +306,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [path], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -330,7 +325,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -349,7 +344,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -371,7 +366,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -392,14 +387,14 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); expect(mockSend).toHaveBeenCalledTimes(3); }); - it('should list and a single object skipping deletion of similar keys of objects', async () => { + it('should list and delete while skipping a single object deletion of a similar key', async () => { const objectKey = 'layer/v1/0/0.png'; const page1Keys = [`${objectKey}8`]; const page2Keys = [objectKey]; @@ -414,7 +409,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [objectKey], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -446,7 +441,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH1, PATH2], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(6); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -466,7 +461,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -481,7 +476,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -500,7 +495,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -519,7 +514,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [] }); + expect(result).toEqual({ failures: new Map() }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -590,7 +585,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [{ path: 'layer/v1/0/0.png', reason: 'AccessDenied' }] }); + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'layer/v1/0/0.png' }]]) }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); @@ -612,7 +607,7 @@ describe('S3StorageProvider', () => { const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); - expect(result).toEqual({ failures: [{ path: 'layer/v1/0/0.png', reason: 'NetworkError' }] }); + expect(result).toEqual({ failures: new Map([['NetworkError', { count: 1, sample: 'layer/v1/0/0.png' }]]) }); expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); diff --git a/tests/strategies/deleteStoredResourcesStrategy.spec.ts b/tests/strategies/deleteStoredResourcesStrategy.spec.ts index 302b194..348baa6 100644 --- a/tests/strategies/deleteStoredResourcesStrategy.spec.ts +++ b/tests/strategies/deleteStoredResourcesStrategy.spec.ts @@ -8,8 +8,10 @@ import type { ConfigType } from '@src/common/config'; import { createMockStoredResourcesDeletionStrategyConfig, createMockLogger, createMockStorageProvider } from '../helpers/mocks'; const S3_BUCKET = 'test-bucket'; -const s3Params = { storageProvider: SourceType.S3, paths: ['layer1'], bucket: S3_BUCKET }; -const fsParams = { storageProvider: SourceType.FS, paths: ['layer2'] }; +const FS_SUB_PATH = 'test/artifacts/tiles'; + +const s3Params: DeleteStoredResourcesParams = { storageProvider: SourceType.S3, paths: ['layer1'], bucket: S3_BUCKET }; +const fsParams: DeleteStoredResourcesParams = { storageProvider: SourceType.FS, paths: ['layer2'], subPath: FS_SUB_PATH }; describe('DeleteStoredResourcesStrategy', () => { let strategy: DeleteStoredResourcesStrategy; @@ -79,7 +81,7 @@ describe('DeleteStoredResourcesStrategy', () => { it('should call delete all resources on FS provider', async () => { await strategy.execute(fsParams); - expect(mockFsProvider.deleteResources).toHaveBeenCalledWith({ paths: fsParams.paths, storageProvider: 'FS' }); + expect(mockFsProvider.deleteResources).toHaveBeenCalledWith({ paths: fsParams.paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); }); @@ -115,7 +117,7 @@ describe('DeleteStoredResourcesStrategy', () => { it('should throw RecoverableError when deleteResources returns failures', async () => { (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ - failures: [{ path: 'layer1/0/0.png', reason: 'AccessDenied' }], + failures: new Map([['AccessDenied', { count: 1, sample: 'layer1/0/0.png' }]]), }); const result = strategy.execute(s3Params); diff --git a/tests/strategies/tilesDeletionStrategy.spec.ts b/tests/strategies/tilesDeletionStrategy.spec.ts index 161a2c7..75864bb 100644 --- a/tests/strategies/tilesDeletionStrategy.spec.ts +++ b/tests/strategies/tilesDeletionStrategy.spec.ts @@ -231,7 +231,7 @@ describe('TilesDeletionStrategy', () => { }); it('should not call updateProgress when retryable failures occur', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); @@ -239,7 +239,7 @@ describe('TilesDeletionStrategy', () => { }); it('should not call updateProgress when only not-found failures occur', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); expect(mockUpdateProgress).not.toHaveBeenCalled(); @@ -248,62 +248,61 @@ describe('TilesDeletionStrategy', () => { describe('failure handling', () => { it('should throw RecoverableError when provider returns fatal failed paths', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); }); it('should include fatal failed count in RecoverableError message', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'AccessDenied' }, - { path: tilePath(10, 0, 1), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 2, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 2/); }); it('should include grouped reason counts in RecoverableError message', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'EACCES' }, - { path: tilePath(10, 0, 1), reason: 'EACCES' }, - { path: tilePath(10, 1, 0), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ + failures: new Map([ + ['EACCES', { count: 2, sample: tilePath(10, 0, 0) }], + ['AccessDenied', { count: 1, sample: tilePath(10, 1, 0) }], + ]), + }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Reasons: EACCES=2, AccessDenied=1/); }); it('should exclude not-found reasons from the RecoverableError reason summary', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'NoSuchKey' }, - { path: tilePath(10, 0, 1), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ + failures: new Map([ + ['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }], + ['AccessDenied', { count: 1, sample: tilePath(10, 0, 1) }], + ]), + }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 1.*Reasons: AccessDenied=1/); }); it('should include path and reason in the failure sample', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(/layer\/v1\/10\/0\/0\.png \(AccessDenied\)/); }); it('should resolve successfully when all failures are not-found (ENOENT)', async () => { - vi.mocked(MockFsProvider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'ENOENT' }, - { path: tilePath(10, 0, 1), reason: 'ENOENT' }, - ]); + vi.mocked(MockFsProvider.delete).mockResolvedValue({ + failures: new Map([['ENOENT', { count: 1, sample: tilePath(10, 0, 0) }]]), + }); await expect(strategy.execute(fsParams)).resolves.toBeUndefined(); }); it('should resolve successfully when all failures are not-found (NoSuchKey)', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); }); it('should resolve successfully when provider returns no failed paths', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map() }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); }); From fe69678ac67047f75116e4ca74017ebfb8e2939e Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:54:22 +0300 Subject: [PATCH 52/66] refactor: modify paths validity checks --- .../storageProviders/fsStorageProvider.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index c3963a3..8b08fb2 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -35,6 +35,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { ) { this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; this.subPaths = Object.values(this.fsConfig.subPaths); + if (this.subPaths.length === 0) throw new ConfigurationError('subPaths must have at least 1 entry'); if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); this.basePath = resolveAbsolutePath(this.fsConfig.basePath); this.canDeleteFromFolder(this.basePath); @@ -90,8 +91,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const relativePaths = paths.map((path) => join(subPath, path)); - // Prevent path traversal (i.e. accessing folders above root folder) - if (!this.checkPathTraversal(relativePaths)) + if (!this.arePathsValid(relativePaths)) throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself and must match a valid configured path'); let failures: DeleteFailure = new Map(); @@ -144,16 +144,23 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } } - private checkPathTraversal(paths: string[]): boolean { - this.logger.debug({ msg: 'Checking path traversal', paths }); - const result = paths.every((path) => { + /** + * Preforms several checks on input `paths`. + * Includes a check for path traversal (i.e. accessing folders above root folder) + * @param paths + * @returns boolean whether `paths` are valid and pass all checks + */ + private arePathsValid(paths: string[]): boolean { + this.logger.debug({ msg: 'Checking paths validity', paths }); + const badPaths = paths.filter((path) => { const startsWithAllowedSubPath = this.subPaths.some((subPath) => path.startsWith(subPath)); const absolutePath = resolveAbsolutePath(join(this.basePath, path)); const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.basePath)); - return startsWithAllowedSubPath && startsWithBasePath; + return !(startsWithAllowedSubPath && startsWithBasePath); }); - this.logger.debug({ msg: `Path traversal check ${result ? 'succeeded' : 'failed'}` }); - return result; + const areValid = badPaths.length === 0; + this.logger.debug({ msg: `Paths validity check ${areValid ? 'succeeded' : 'failed'}`, ...(!areValid && { badPaths }) }); + return areValid; } // Attempts to remove any directories that became empty after file deletion. From e749794e9129d576fc5f3533e4e207e032b54c3b Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:54:51 +0300 Subject: [PATCH 53/66] refactor: log fs paths deletion progress --- .../storageProviders/fsStorageProvider.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 8b08fb2..33aef99 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -88,6 +88,8 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { subPath, }: Extract): Promise { this.logger.debug({ msg: 'Starting FS files/dirs deletion', subPath, pathsCount: paths.length }); + let totalDeletedPathsCount = 0, + totalFailedPathsCount = 0; const relativePaths = paths.map((path) => join(subPath, path)); @@ -108,12 +110,25 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { if (result.status === 'rejected') { const fullPath = join(this.basePath, relativePathsChunk[idx]!); const reason = describeError(result.reason); - this.logger.error({ msg: 'Failed to delete layer directory', fullPath, reason, err: result.reason }); + this.logger.error({ msg: 'Failed to delete file/folder', fullPath, reason, err: result.reason }); const chunkFailure = chunkFailures.get(reason); chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? fullPath }); } } failures = mergeFailures({ source: chunkFailures, target: failures }); + + let failedPathsCount = 0; + chunkFailures.forEach((chunkFailure) => (failedPathsCount += chunkFailure.count)); + const deletedPathsCount = relativePathsChunk.length - failedPathsCount; + totalDeletedPathsCount += deletedPathsCount; + totalFailedPathsCount += failedPathsCount; + this.logger.debug({ + msg: 'Completed processing current chunk of paths', + deletedPathsCount, + totalDeletedPathsCount, + failedPathsCount, + totalFailedPathsCount, + }); } return { failures }; From 51dc59f0d6d251334d7a5fc096fe913c5a729234 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:28:33 +0300 Subject: [PATCH 54/66] chore: fix config of fs --- config/default.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/default.json b/config/default.json index 4d9c909..1508c6a 100644 --- a/config/default.json +++ b/config/default.json @@ -97,8 +97,10 @@ "delete": { "batchSize": 1000 }, - "basePath": "/tiles", - "subPaths": {} + "basePath": "/data", + "subPaths": { + "tilesSubPath": "tiles" + } } }, "strategies": { From 609f4daa520651493cce7d628d28db678f9d1c08 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:31:38 +0300 Subject: [PATCH 55/66] fix: tiles deletion strategy should include configured sub path --- config/custom-environment-variables.json | 1 + config/default.json | 1 + helm/templates/configmap.yaml | 2 ++ helm/values.yaml | 3 +-- src/cleaner/strategies/tilesDeletionStrategy.ts | 5 ++++- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 9c3a880..1ba7871 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -115,6 +115,7 @@ "strategies": { "tilesDeletion": { "s3Bucket": "TILES_DELETION_S3_BUCKET", + "fsSubPath": "TILES_DELETION_FS_SUB_PATH", "batchSize": { "__name": "TILES_DELETION_BATCH_SIZE", "__format": "number" diff --git a/config/default.json b/config/default.json index 1508c6a..4d97d6d 100644 --- a/config/default.json +++ b/config/default.json @@ -106,6 +106,7 @@ "strategies": { "tilesDeletion": { "s3Bucket": "", + "fsSubPath": "tiles", "batchSize": 1000, "concurrency": 10 } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index abc81c9..27a5aff 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -5,6 +5,7 @@ {{- $fs := ($storage.fs) | default dict -}} {{- $internalPvc := (($fs).internalPvc) | default dict -}} {{- $fsBasePath := clean (printf "/%s" $internalPvc.mountPath) -}} +{{- $fsTilesDeletionSubPath := clean (printf "%s" ($internalPvc.tilesSubPath | default "tiles")) -}} {{- if .Values.enabled -}} apiVersion: v1 kind: ConfigMap @@ -73,6 +74,7 @@ data: {{- end -}} {{- end }} FS_SUB_PATHS: {{ $subPaths | toJson | quote }} + TILES_DELETION_FS_SUB_PATH: {{ $fsTilesDeletionSubPath | quote }} {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} {{- with .Values.env.strategies.tilesDeletion }} diff --git a/helm/values.yaml b/helm/values.yaml index e225e2d..fe496fb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -32,8 +32,7 @@ storage: enabled: false name: "" mountPath: "" - tilesSubPath: "" - + tilesSubPath: "" # e.g. folder/tiles mclabels: component: backend diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index 002c473..ee70f3e 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -1,3 +1,4 @@ +import { join } from 'node:path'; import { NoSuchKey } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; @@ -27,6 +28,7 @@ export class TilesDeletionStrategy implements ITaskStrategy private readonly concurrency: number; private readonly s3Bucket: string; private readonly fsBasePath: string; + private readonly fsTilesDeletionSubPath: string; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @@ -39,6 +41,7 @@ export class TilesDeletionStrategy implements ITaskStrategy this.concurrency = config.get('strategies.tilesDeletion.concurrency') as unknown as number; this.s3Bucket = config.get('strategies.tilesDeletion.s3Bucket') as unknown as string; this.fsBasePath = config.get('storage.fs.basePath') as unknown as FsConfig['basePath']; + this.fsTilesDeletionSubPath = config.get('strategies.tilesDeletion.fsSubPath') as unknown as string; } public validate(params: unknown): TilesDeletionParams { @@ -119,7 +122,7 @@ export class TilesDeletionStrategy implements ITaskStrategy // eslint-disable-next-line @typescript-eslint/naming-convention const storageProvider = this.storageProviders[params.sourceProvider]; if (storageProvider === undefined) throw new UnrecoverableError(`Unsupported storage provider ${params.sourceProvider}`); - const storageTarget = params.sourceProvider === SourceType.S3 ? this.s3Bucket : this.fsBasePath; + const storageTarget = params.sourceProvider === SourceType.S3 ? this.s3Bucket : join(this.fsBasePath, this.fsTilesDeletionSubPath); this.logger.debug({ msg: `Using ${params.sourceProvider} provider` }); return { provider: storageProvider, storageTarget }; } From 0e93e2d523c1f578455fa9864f873612fa48c336 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:34:17 +0300 Subject: [PATCH 56/66] test: modify tests and setup to support fs sub path --- tests/helpers/mocks.ts | 10 ++++++---- tests/storageProviders/fsStorageProvider.spec.ts | 2 +- tests/strategies/tilesDeletionStrategy.spec.ts | 7 ++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 300ea03..07aa842 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -82,7 +82,8 @@ export const TILES_DELETION_CONFIG_DEFAULTS = { batchSize: 100, concurrency: 2, s3Bucket: 'test-bucket', - fsBasePath: '/test/tiles', + fsBasePath: '/test', + fsSubPath: 'tiles', } as const; export function createMockStrategyConfig(overrides: Record = {}): ConfigType { @@ -90,7 +91,8 @@ export function createMockStrategyConfig(overrides: Record = {} 'strategies.tilesDeletion.batchSize': TILES_DELETION_CONFIG_DEFAULTS.batchSize, 'strategies.tilesDeletion.concurrency': TILES_DELETION_CONFIG_DEFAULTS.concurrency, 'strategies.tilesDeletion.s3Bucket': TILES_DELETION_CONFIG_DEFAULTS.s3Bucket, - 'storage.fs.basePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, + 'strategies.tilesDeletion.fsSubPath': TILES_DELETION_CONFIG_DEFAULTS.fsSubPath, + 'storage.fs.basePath': FS_STORAGE_CONFIG_DEFAULTS.basePath, ...overrides, }; return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; @@ -133,9 +135,9 @@ export const FS_STORAGE_CONFIG_DEFAULTS = { delete: { batchSize: 3, }, - basePath: '/test/tiles', + basePath: '/test', subPaths: { - tiles: 'test/artifacts/tiles', + tiles: 'artifacts/tiles', }, } as const satisfies FsConfig; diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index 598f026..3f07c4b 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -208,7 +208,7 @@ describe('FsStorageProvider', () => { }); describe('#deleteResources', () => { - const FS_SUB_PATH = 'test/artifacts/tiles'; + const FS_SUB_PATH = FS_STORAGE_CONFIG_DEFAULTS.subPaths.tiles; const RELATIVE_PATH = 'layer/v1'; it('should successfully call delete all files and return without failures', async () => { diff --git a/tests/strategies/tilesDeletionStrategy.spec.ts b/tests/strategies/tilesDeletionStrategy.spec.ts index 75864bb..31e7c69 100644 --- a/tests/strategies/tilesDeletionStrategy.spec.ts +++ b/tests/strategies/tilesDeletionStrategy.spec.ts @@ -1,3 +1,4 @@ +import { join } from 'node:path'; import { faker } from '@faker-js/faker'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; @@ -8,7 +9,7 @@ import type { TaskContext } from '@src/cleaner/strategies/strategyFactory'; import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from '../helpers/mocks'; -const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH } = TILES_DELETION_CONFIG_DEFAULTS; +const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH, fsSubPath: FS_SUB_PATH } = TILES_DELETION_CONFIG_DEFAULTS; const JOB_ID = faker.string.uuid(); const TASK_ID = faker.string.uuid(); @@ -114,7 +115,7 @@ describe('TilesDeletionStrategy', () => { it('should check targetExists with FS base path and tilesPath as relativePath', async () => { await strategy.execute(fsParams); - expect(MockFsProvider.targetExists).toHaveBeenCalledWith(FS_BASE_PATH, fsParams.tilesPath); + expect(MockFsProvider.targetExists).toHaveBeenCalledWith(join(FS_BASE_PATH, FS_SUB_PATH), fsParams.tilesPath); }); }); @@ -129,7 +130,7 @@ describe('TilesDeletionStrategy', () => { it('should call FS provider with fsBasePath as storage target', async () => { await strategy.execute(fsParams); - expect(MockFsProvider.delete).toHaveBeenCalledWith(expect.any(Array), FS_BASE_PATH); + expect(MockFsProvider.delete).toHaveBeenCalledWith(expect.any(Array), join(FS_BASE_PATH, FS_SUB_PATH)); expect(MockS3Provider.delete).not.toHaveBeenCalled(); }); From acd254977e8ad7d990aab6b3832631ce44efc4ce Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:47:34 +0300 Subject: [PATCH 57/66] refactor: enrich log message with example page response --- src/cleaner/storageProviders/s3StorageProvider.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 8e2e773..1b9c88e 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -187,6 +187,7 @@ export class S3StorageProvider implements IStorageProvider 0 && { samplePageResponse: pageOfObjects[0] }), }); const keys = pageOfObjects.map((obj) => obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); From a33d51494afea375e11f8ecad9d339e388689bd9 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:09:42 +0300 Subject: [PATCH 58/66] refactor: pr comment rename deletion result interface --- src/cleaner/storageProviders/failuresHandling.ts | 6 +++--- src/cleaner/storageProviders/fsStorageProvider.ts | 12 +++--------- src/cleaner/storageProviders/iStorageProvider.ts | 6 +++--- src/cleaner/storageProviders/index.ts | 2 +- src/cleaner/storageProviders/s3StorageProvider.ts | 12 +++--------- 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/cleaner/storageProviders/failuresHandling.ts b/src/cleaner/storageProviders/failuresHandling.ts index 6c6c4ed..19c1d1d 100644 --- a/src/cleaner/storageProviders/failuresHandling.ts +++ b/src/cleaner/storageProviders/failuresHandling.ts @@ -1,4 +1,4 @@ -import type { DeleteFailure, DeleteResourcesResult } from './iStorageProvider'; +import type { DeleteFailure, DeleteResult } from './iStorageProvider'; /** * Aggregated view of a batch of delete failures — designed to be embedded in a @@ -27,10 +27,10 @@ export const mergeFailures = ({ source, target }: { source: DeleteFailure; targe /** * Reduces a list of provider delete failures into a compact, log-friendly * shape. Lives alongside `IStorageProvider` because it operates purely on - * `DeleteResourcesResult` — any caller of `provider.delete()` can use it, regardless + * `DeleteResult` — any caller of `provider.delete()` can use it, regardless * of which storage backend produced the failures. */ -export function summarizeDeleteFailures({ failures }: DeleteResourcesResult): DeleteFailureSummary { +export function summarizeDeleteFailures({ failures }: DeleteResult): DeleteFailureSummary { let failuresCount = 0; for (const { count } of failures.values()) { failuresCount += count; diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 33aef99..f2b37b7 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -3,13 +3,7 @@ import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; -import { - mergeFailures, - type DeleteFailure, - type DeleteResourcesResult, - type IStorageProvider, - type StorageProvider, -} from '@src/cleaner/storageProviders'; +import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; import type { ConfigType } from '@src/common/config'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; @@ -53,7 +47,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { } } - public async delete(paths: string[], basePath: string): Promise { + public async delete(paths: string[], basePath: string): Promise { this.logger.debug({ msg: 'Deleting files from filesystem', basePath, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); @@ -86,7 +80,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { public async deleteResources({ paths, subPath, - }: Extract): Promise { + }: Extract): Promise { this.logger.debug({ msg: 'Starting FS files/dirs deletion', subPath, pathsCount: paths.length }); let totalDeletedPathsCount = 0, totalFailedPathsCount = 0; diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index 509ece1..481a529 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -5,7 +5,7 @@ import type { DeleteStoredResourcesParams, Storage } from '@map-colonies/raster- */ export type DeleteFailure = Map; -export interface DeleteResourcesResult { +export interface DeleteResult { failures: DeleteFailure; } @@ -20,14 +20,14 @@ export interface IStorageProvider { * Returns an object including delete failures aggregation with one entry per failed reason. * "Not found" is reported as a failure with additional metadata on failure - count and sample */ - delete: (paths: string[], storageTarget: string) => Promise; + delete: (paths: string[], storageTarget: string) => Promise; /** * Deletes ALL objects/files under the given paths. * - S3: storageTarget = bucket name; paths are root paths to resource(s) * - FS: storageTarget = base directory; paths are relative paths from mounted dir */ - deleteResources: (deleteStoredResourcesParams: Extract) => Promise; + deleteResources: (deleteStoredResourcesParams: Extract) => Promise; /** * Returns true if relativePath exists within storageTarget and contains data. diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 070569e..5160586 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,4 @@ export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './failuresHandling'; export { FsStorageProvider, type FsConfig } from './fsStorageProvider'; -export type { DeleteFailure, DeleteResourcesResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; +export type { DeleteFailure, DeleteResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 1b9c88e..8612363 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -15,13 +15,7 @@ import { import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; import type { ConfigType } from '@common/config'; -import { - mergeFailures, - type DeleteFailure, - type DeleteResourcesResult, - type IStorageProvider, - type StorageProvider, -} from '@src/cleaner/storageProviders'; +import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; @@ -67,7 +61,7 @@ export class S3StorageProvider implements IStorageProvider { + public async delete(paths: string[], bucket: string): Promise { this.logger.debug({ msg: 'Deleting objects from S3', bucket, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); @@ -82,7 +76,7 @@ export class S3StorageProvider implements IStorageProvider): Promise { + }: Extract): Promise { this.logger.debug({ msg: `Starting S3 resources deletion`, bucket, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); From b52f5d5d6db674865b8a39025f92fd0997183f87 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:00:01 +0300 Subject: [PATCH 59/66] fix: enhance path validation for sub paths --- src/cleaner/storageProviders/fsStorageProvider.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index f2b37b7..3518069 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -88,7 +88,9 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { const relativePaths = paths.map((path) => join(subPath, path)); if (!this.arePathsValid(relativePaths)) - throw new UnrecoverableError('Cannot delete files/folders outside base path or base path itself and must match a valid configured path'); + throw new UnrecoverableError( + 'Cannot delete files/folders outside base path or subpath as well as base path or subpath itself. paths must also match a valid configured path.' + ); let failures: DeleteFailure = new Map(); @@ -162,7 +164,7 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private arePathsValid(paths: string[]): boolean { this.logger.debug({ msg: 'Checking paths validity', paths }); const badPaths = paths.filter((path) => { - const startsWithAllowedSubPath = this.subPaths.some((subPath) => path.startsWith(subPath)); + const startsWithAllowedSubPath = this.subPaths.some((subPath) => path.startsWith(normalizeFolderPath(subPath))); const absolutePath = resolveAbsolutePath(join(this.basePath, path)); const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.basePath)); return !(startsWithAllowedSubPath && startsWithBasePath); From 44470c5bfe88b8523770c7bc32a9949a8ead8a6d Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:15:30 +0300 Subject: [PATCH 60/66] fix: deletion count calculation in TilesDeletionStrategy --- src/cleaner/strategies/tilesDeletionStrategy.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index ee70f3e..d2978d6 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -84,7 +84,11 @@ export class TilesDeletionStrategy implements ITaskStrategy (NOT_FOUND_REASONS.has(failure[0]) ? notFound : retryable).set(failure[0], failure[1]); } - const deletedCount = totalTiles - retryable.size - notFound.size; + let retryableCount = 0; + retryable.forEach((retryableFailure) => (retryableCount += retryableFailure.count)); + let notFoundCount = 0; + notFound.forEach((notFoundFailure) => (notFoundCount += notFoundFailure.count)); + const deletedCount = totalTiles - retryableCount - notFoundCount; if (retryable.size > 0) { const { failuresCount, samples, summary } = summarizeDeleteFailures({ failures: retryable }); @@ -92,7 +96,7 @@ export class TilesDeletionStrategy implements ITaskStrategy msg: 'Tiles deletion partially failed', totalTiles, uniqueFailureTypesCount: retryable.size, - notFoundCount: notFound.size, + notFoundCount, deletedCount, totalFailuresCount: failuresCount, summary, From f7e91268a54adcd000cb2374779465e86426d84e Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:20:17 +0300 Subject: [PATCH 61/66] fix: track processed tiles count during batch deletion --- src/cleaner/strategies/tilesDeletionStrategy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index d2978d6..792b884 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -163,7 +163,8 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); } if (pendingBatches.length > 0) { - const { batchFailures } = await this.flushBatches(provider, storageTarget, pendingBatches); + const { batchFailures, processedTilesCount } = await this.flushBatches(provider, storageTarget, pendingBatches); + processedTiles += processedTilesCount; failures = mergeFailures({ source: batchFailures, target: failures }); this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.size }); } From 1b86ddc1cb226471d236389a215b4d1787ece6a6 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:59:09 +0300 Subject: [PATCH 62/66] refactor: remove chunking since it is also done at tileDeletionStrategy level --- .../storageProviders/fsStorageProvider.ts | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 3518069..1928c90 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -51,26 +51,24 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { this.logger.debug({ msg: 'Deleting files from filesystem', basePath, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); - for (const relativePaths of getChunk(paths, this.fsConfig.delete.batchSize)) { - const results = await Promise.allSettled( - relativePaths.map(async (relativePath) => { - await unlink(join(basePath, relativePath)); - }) - ); - - const chunkFailures: DeleteFailure = new Map(); - for (const [idx, result] of results.entries()) { - if (result.status === 'rejected') { - const relativePath = relativePaths[idx]!; - const error: unknown = result.reason; - const reason = describeError(error); - this.logger.debug({ msg: 'Failed to delete file', path: join(basePath, relativePath), reason, error }); - const chunkFailure = chunkFailures.get(reason); - chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePaths[idx]! }); - } + const results = await Promise.allSettled( + paths.map(async (relativePath) => { + await unlink(join(basePath, relativePath)); + }) + ); + + const chunkFailures: DeleteFailure = new Map(); + for (const [idx, result] of results.entries()) { + if (result.status === 'rejected') { + const relativePath = paths[idx]!; + const error: unknown = result.reason; + const reason = describeError(error); + this.logger.debug({ msg: 'Failed to delete file', path: join(basePath, relativePath), reason, error }); + const chunkFailure = chunkFailures.get(reason); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePath }); } - failures = mergeFailures({ source: chunkFailures, target: failures }); } + failures = mergeFailures({ source: chunkFailures, target: failures }); await this.cleanupEmptyDirs(paths, basePath); From f3021c851ad4947b24099b897d9f799ac3ca3a5b Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:38:22 +0300 Subject: [PATCH 63/66] refactor: update FS and S3 storage providers to use DI and modify configuration handling --- .../storageProviders/fsStorageProvider.ts | 69 +++----------- src/cleaner/storageProviders/index.ts | 10 +- .../storageProviders/s3StorageProvider.ts | 52 ++++------- src/cleaner/storageProviders/storageConfig.ts | 92 +++++++++++++++++++ src/cleaner/utils/fs.ts | 28 ++++++ src/cleaner/utils/index.ts | 3 +- src/common/constants.ts | 2 + src/containerConfig.ts | 16 ++-- 8 files changed, 175 insertions(+), 97 deletions(-) create mode 100644 src/cleaner/storageProviders/storageConfig.ts create mode 100644 src/cleaner/utils/fs.ts diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 1928c90..f91e69b 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -1,39 +1,23 @@ -import { accessSync, constants, statSync } from 'node:fs'; import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; -import type { ConfigType } from '@src/common/config'; -import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; +import { SERVICES } from '@common/constants'; +import { describeError, UnrecoverableError } from '../errors'; +import type { FsStorageConfig } from './storageConfig'; type FSStorageProviderType = Extract; -export interface FsConfig { - delete: { - batchSize: number; - }; - basePath: string; - subPaths: Record; -} - +@injectable() export class FsStorageProvider implements IStorageProvider<'FS'> { - private readonly fsConfig: FsConfig; - private readonly basePath: string; - private readonly subPaths: string[]; - public constructor( - private readonly config: ConfigType, - private readonly logger: Logger + @inject(SERVICES.FS_STORAGE_CONFIG) private readonly fsConfig: FsStorageConfig, + @inject(SERVICES.LOGGER) private readonly logger: Logger ) { - this.fsConfig = this.config.get('storage.fs') as unknown as FsConfig; - this.subPaths = Object.values(this.fsConfig.subPaths); - if (this.subPaths.length === 0) throw new ConfigurationError('subPaths must have at least 1 entry'); - if (this.fsConfig.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); - this.basePath = resolveAbsolutePath(this.fsConfig.basePath); - this.canDeleteFromFolder(this.basePath); - this.logger.debug({ msg: 'Loaded FS storage provider', basePath: this.basePath }); + this.logger.debug({ msg: 'Loaded FS storage provider', basePath: this.fsConfig.basePath }); } public async targetExists(basePath: string, relativePath: string): Promise { @@ -92,17 +76,17 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { let failures: DeleteFailure = new Map(); - for (const relativePathsChunk of getChunk(relativePaths, this.fsConfig.delete.batchSize)) { + for (const relativePathsChunk of getChunk(relativePaths, this.fsConfig.batchSize)) { const results = await Promise.allSettled( relativePathsChunk.map(async (relativePath) => { - await rm(join(this.basePath, relativePath), { recursive: true, force: true }); + await rm(join(this.fsConfig.basePath, relativePath), { recursive: true, force: true }); }) ); const chunkFailures: DeleteFailure = new Map(); for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { - const fullPath = join(this.basePath, relativePathsChunk[idx]!); + const fullPath = join(this.fsConfig.basePath, relativePathsChunk[idx]!); const reason = describeError(result.reason); this.logger.error({ msg: 'Failed to delete file/folder', fullPath, reason, err: result.reason }); const chunkFailure = chunkFailures.get(reason); @@ -128,31 +112,6 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { return { failures }; } - private canDeleteFromFolder(path: string): void { - try { - accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); - this.logger.debug({ msg: 'Able to delete from directory', path }); - } catch (err) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { - throw new ConfigurationError(`FS path does not exist: ${path}`); - } else if (err instanceof Error && 'code' in err && (err.code === 'EACCES' || err.code === 'EPERM')) { - throw new ConfigurationError(`FS path permission denied for path: ${path}`); - } else { - throw new ConfigurationError(`An unexpected error occurred on FS path accessibility check: ${describeError(err)}`); - } - } - - try { - const pathStat = statSync(path); - if (!pathStat.isDirectory()) { - throw new ConfigurationError(`FS path exists but it is a file, not a directory: ${path}`); - } - } catch (err) { - if (err instanceof ConfigurationError) throw err; - throw new ConfigurationError(`An unexpected error occurred on FS info check: ${describeError(err)}`); - } - } - /** * Preforms several checks on input `paths`. * Includes a check for path traversal (i.e. accessing folders above root folder) @@ -162,9 +121,9 @@ export class FsStorageProvider implements IStorageProvider<'FS'> { private arePathsValid(paths: string[]): boolean { this.logger.debug({ msg: 'Checking paths validity', paths }); const badPaths = paths.filter((path) => { - const startsWithAllowedSubPath = this.subPaths.some((subPath) => path.startsWith(normalizeFolderPath(subPath))); - const absolutePath = resolveAbsolutePath(join(this.basePath, path)); - const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.basePath)); + const startsWithAllowedSubPath = this.fsConfig.subPaths.some((subPath) => path.startsWith(normalizeFolderPath(subPath))); + const absolutePath = resolveAbsolutePath(join(this.fsConfig.basePath, path)); + const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.fsConfig.basePath)); return !(startsWithAllowedSubPath && startsWithBasePath); }); const areValid = badPaths.length === 0; diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 5160586..43f390e 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,12 @@ export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './failuresHandling'; -export { FsStorageProvider, type FsConfig } from './fsStorageProvider'; +export { FsStorageProvider } from './fsStorageProvider'; export type { DeleteFailure, DeleteResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; +export { + buildFsStorageConfig, + buildS3StorageConfig, + type FsConfig, + type FsStorageConfig, + type S3Config, + type S3StorageConfig, +} from './storageConfig'; diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 8612363..9cccf2c 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -14,58 +14,42 @@ import { } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; -import type { ConfigType } from '@common/config'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '@common/constants'; import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; -import { ConfigurationError, describeError, UnrecoverableError } from '../errors'; +import { describeError, UnrecoverableError } from '../errors'; +import type { S3StorageConfig } from './storageConfig'; type S3StorageProviderType = Extract; -// S3/MinIO reject DeleteObjects requests with more than 1000 keys, regardless of configured batch size. -const S3_DELETE_OBJECTS_MAX_KEYS = 1000; - -export interface S3Config { - delete: { - batchSize: number; - }; - endpoint: string; - accessKeyId: string; - secretAccessKey: string; - sslEnabled?: boolean; - forcePathStyle?: boolean; - region?: string; -} - +@injectable() export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; - private readonly s3Config: S3Config; - private readonly batchSize: number; public constructor( - config: ConfigType, - private readonly logger: Logger + @inject(SERVICES.S3_STORAGE_CONFIG) private readonly s3Config: S3StorageConfig, + @inject(SERVICES.LOGGER) private readonly logger: Logger ) { - this.s3Config = config.get('storage.s3') as unknown as S3Config; - if (this.s3Config.delete.batchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); - this.batchSize = Math.min(this.s3Config.delete.batchSize, S3_DELETE_OBJECTS_MAX_KEYS); + // TODO: move client to a singleton resolution since this.s3Client = new S3Client({ - endpoint: this.s3Config.endpoint, + endpoint: s3Config.endpoint, credentials: { - accessKeyId: this.s3Config.accessKeyId, - secretAccessKey: this.s3Config.secretAccessKey, + accessKeyId: s3Config.accessKeyId, + secretAccessKey: s3Config.secretAccessKey, }, - forcePathStyle: this.s3Config.forcePathStyle, - region: this.s3Config.region, - tls: this.s3Config.sslEnabled, + forcePathStyle: s3Config.forcePathStyle, + region: s3Config.region, + tls: s3Config.sslEnabled, }); - this.logger.debug({ msg: 'Loaded S3 storage provider', endpoint: this.s3Config.endpoint, batchSize: this.batchSize }); + this.logger.debug({ msg: 'Loaded S3 storage provider', endpoint: s3Config.endpoint, batchSize: this.s3Config.batchSize }); } public async delete(paths: string[], bucket: string): Promise { this.logger.debug({ msg: 'Deleting objects from S3', bucket, pathsCount: paths.length }); let failures: DeleteFailure = new Map(); - for (const chunk of getChunk(paths, Math.min(this.s3Config.delete.batchSize, S3_DELETE_OBJECTS_MAX_KEYS))) { + for (const chunk of getChunk(paths, this.s3Config.batchSize)) { const chunkFailures = await this.deleteObjects(chunk, bucket); failures = mergeFailures({ source: chunkFailures, target: failures }); } @@ -170,7 +154,7 @@ export class S3StorageProvider implements IStorageProvider 0 && { samplePageResponse: pageOfObjects[0] }), }); const keys = pageOfObjects.map((obj) => obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); diff --git a/src/cleaner/storageProviders/storageConfig.ts b/src/cleaner/storageProviders/storageConfig.ts new file mode 100644 index 0000000..8bd2f8b --- /dev/null +++ b/src/cleaner/storageProviders/storageConfig.ts @@ -0,0 +1,92 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { assertCanDeleteFromFolder, resolveAbsolutePath } from '@src/cleaner/utils'; +import type { ConfigType } from '@src/common/config'; +import { ConfigurationError } from '../errors'; + +// S3/MinIO reject DeleteObjects requests with more than 1000 keys, regardless of configured batch size. +const S3_DELETE_OBJECTS_MAX_KEYS = 1000; + +export interface FsConfig { + delete: { + batchSize: number; + }; + basePath: string; + subPaths: Record; +} + +export interface S3Config { + delete: { + batchSize: number; + }; + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + sslEnabled?: boolean; + forcePathStyle?: boolean; + region?: string; +} + +export interface FsStorageConfig { + basePath: string; + subPaths: string[]; + batchSize: number; +} + +export interface S3StorageConfig { + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + sslEnabled?: boolean; + forcePathStyle?: boolean; + region?: string; + batchSize: number; +} + +/** + * Reads and validates `storage.fs`. + * @returns {FsStorageConfig} FS configuration for FsStorageProvider + * @throws {ConfigurationError} if the config is unusable or the base path is not a writable directory + */ +export function buildFsStorageConfig(config: ConfigType, logger: Logger): FsStorageConfig { + //TODO: when we create a worker config schema the shape checks below can be dropped along with the cast + const fsConfig = config.get('storage.fs') as unknown as FsConfig; + + const { + delete: { batchSize: deleteBatchSize }, + ...fsStorageConfig + } = fsConfig; + + const subPaths = Object.values(fsStorageConfig.subPaths); + if (subPaths.length === 0) throw new ConfigurationError('Deletion subpaths must have at least 1 entry'); + if (deleteBatchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); + + const basePath = resolveAbsolutePath(fsStorageConfig.basePath); + assertCanDeleteFromFolder(basePath, logger); + + logger.info({ msg: 'Validated FS storage config', basePath, subPaths, batchSize: deleteBatchSize }); + return { basePath, subPaths, batchSize: deleteBatchSize }; +} + +/** + * Reads and validates `storage.s3`. + * @returns {S3StorageConfig} S3 configuration for S3StorageProvider + * @throws {ConfigurationError} if the config is unusable + */ +export function buildS3StorageConfig(config: ConfigType, logger: Logger): S3StorageConfig { + //TODO: when we create a worker config schema the shape check below can be dropped along with the cast + const s3Config = config.get('storage.s3') as unknown as S3Config; + + const { + delete: { batchSize: deleteBatchSize }, + ...s3StorageConfig + } = s3Config; + + if (deleteBatchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); + const batchSize = Math.min(deleteBatchSize, S3_DELETE_OBJECTS_MAX_KEYS); + + logger.debug({ msg: 'Validated S3 storage config', endpoint: s3StorageConfig.endpoint, batchSize }); + return { + ...s3StorageConfig, + batchSize, + }; +} diff --git a/src/cleaner/utils/fs.ts b/src/cleaner/utils/fs.ts new file mode 100644 index 0000000..18a6c2a --- /dev/null +++ b/src/cleaner/utils/fs.ts @@ -0,0 +1,28 @@ +import { accessSync, constants, statSync } from 'node:fs'; +import type { Logger } from '@map-colonies/js-logger'; +import { ConfigurationError, describeError } from '../errors'; + +export const assertCanDeleteFromFolder = (path: string, logger: Logger): void => { + try { + accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); + logger.debug({ msg: 'Able to delete from directory', path }); + } catch (err) { + if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { + throw new ConfigurationError(`FS path does not exist: ${path}`); + } else if (err instanceof Error && 'code' in err && (err.code === 'EACCES' || err.code === 'EPERM')) { + throw new ConfigurationError(`FS path permission denied for path: ${path}`); + } else { + throw new ConfigurationError(`An unexpected error occurred on FS path accessibility check: ${describeError(err)}`); + } + } + + try { + const pathStat = statSync(path); + if (!pathStat.isDirectory()) { + throw new ConfigurationError(`FS path exists but it is a file, not a directory: ${path}`); + } + } catch (err) { + if (err instanceof ConfigurationError) throw err; + throw new ConfigurationError(`An unexpected error occurred on FS info check: ${describeError(err)}`); + } +}; diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index 9d2809c..49d0a4f 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,4 +1,5 @@ export { getChunk } from './chunk'; +export { assertCanDeleteFromFolder } from './fs'; export { buildPollingPairs } from './pairBuilder'; -export { validateSchema } from './validationHelper'; export { normalizeFolderPath, resolveAbsolutePath } from './path'; +export { validateSchema } from './validationHelper'; diff --git a/src/common/constants.ts b/src/common/constants.ts index 01c8bfd..a9be61f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -19,6 +19,8 @@ export const SERVICES = { TASK_VALIDATOR: Symbol('TaskValidator'), POLLING_PAIRS: Symbol('PollingPairs'), STORAGE_PROVIDERS: Symbol('StorageProviders'), + FS_STORAGE_CONFIG: Symbol('FsStorageConfig'), + S3_STORAGE_CONFIG: Symbol('S3StorageConfig'), TASK_CONTEXT: Symbol('TaskContext'), JOB_TRACKER_CLIENT: Symbol('JobTrackerClient'), // ============================================================================= diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 41ee7fb..db8c797 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -13,7 +13,7 @@ import { getTracing } from '@common/tracing'; import type { StorageProviders } from '@src/cleaner/storageProviders'; import { ErrorHandler } from './cleaner/errors'; import { JobTrackerClient } from './cleaner/httpClients'; -import { FsStorageProvider, S3StorageProvider } from './cleaner/storageProviders'; +import { buildFsStorageConfig, buildS3StorageConfig, FsStorageProvider, S3StorageProvider } from './cleaner/storageProviders'; import { DeleteStoredResourcesStrategy, StrategyFactory, TilesDeletionStrategy } from './cleaner/strategies'; import type { QueueConfig } from './cleaner/types'; import { ConfigType, getConfig } from './common/config'; @@ -35,6 +35,11 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const metricsRegistry = new Registry(); configInstance.initializeMetrics(metricsRegistry); + // Startup validations + const cleanupStorageProviders = configInstance.get('storage.cleanupStorageProviders') as unknown as string[]; + const fsStorageConfig = cleanupStorageProviders.includes(SourceType.FS) ? buildFsStorageConfig(configInstance, logger) : undefined; + const s3StorageConfig = cleanupStorageProviders.includes(SourceType.S3) ? buildS3StorageConfig(configInstance, logger) : undefined; + const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: configInstance } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -98,16 +103,15 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise useClass: JobTrackerClient, }, }, + ...(fsStorageConfig ? [{ token: SERVICES.FS_STORAGE_CONFIG, provider: { useValue: fsStorageConfig } }] : []), + ...(s3StorageConfig ? [{ token: SERVICES.S3_STORAGE_CONFIG, provider: { useValue: s3StorageConfig } }] : []), { token: SERVICES.STORAGE_PROVIDERS, provider: { useFactory: instancePerContainerCachingFactory((container) => { - const config = container.resolve(SERVICES.CONFIG); - const logger = container.resolve(SERVICES.LOGGER); - const cleanupStorageProviders = config.get('storage.cleanupStorageProviders') as unknown as string[]; const providers = { - ...(cleanupStorageProviders.includes(SourceType.S3) && { [SourceType.S3]: new S3StorageProvider(config, logger) }), - ...(cleanupStorageProviders.includes(SourceType.FS) && { [SourceType.FS]: new FsStorageProvider(config, logger) }), + ...(s3StorageConfig && { [SourceType.S3]: container.resolve(S3StorageProvider) }), + ...(fsStorageConfig && { [SourceType.FS]: container.resolve(FsStorageProvider) }), }; return providers; }), From d106d79abe3470af7a28662727baa3263a816cd1 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:33:56 +0300 Subject: [PATCH 64/66] refactor: should short-circuit empty paths input --- src/cleaner/storageProviders/s3StorageProvider.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 9cccf2c..57be934 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -64,6 +64,7 @@ export class S3StorageProvider implements IStorageProvider path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion const exists = await this.bucketExists(bucket); From df485bdf0dfdfad0119084227e6a64c1f5430eca Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:35:48 +0300 Subject: [PATCH 65/66] test: update test files with config setup validations and cover edge cases --- tests/helpers/mocks.ts | 31 ++- .../fsStorageProvider.spec.ts | 215 +++++++++++++----- .../s3StorageProvider.spec.ts | 180 +++++++++++---- tests/storageProviders/storageConfig.spec.ts | 136 +++++++++++ .../deleteStoredResourcesStrategy.spec.ts | 29 +++ .../strategies/tilesDeletionStrategy.spec.ts | 107 ++++++++- tests/utils/fs.spec.ts | 97 ++++++++ 7 files changed, 689 insertions(+), 106 deletions(-) create mode 100644 tests/storageProviders/storageConfig.spec.ts create mode 100644 tests/utils/fs.spec.ts diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 07aa842..949b565 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -1,9 +1,8 @@ import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { vi } from 'vitest'; -import type { FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; import type { IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders/iStorageProvider'; -import type { S3Config } from '@src/cleaner/storageProviders/s3StorageProvider'; +import type { FsConfig, FsStorageConfig, S3Config, S3StorageConfig } from '@src/cleaner/storageProviders/storageConfig'; import type { ErrorHandler } from '../../src/cleaner/errors'; import type { JobTrackerClient } from '../../src/cleaner/httpClients'; import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; @@ -129,6 +128,22 @@ export function createMockS3Config(overrides: Record = {}): Con } as unknown as ConfigType; } +// ─── S3 Validated Storage Config ─────────────────────────────────── + +export const S3_VALIDATED_CONFIG_DEFAULTS = { + endpoint: S3_STORAGE_CONFIG_DEFAULTS.endpoint, + accessKeyId: S3_STORAGE_CONFIG_DEFAULTS.accessKeyId, + secretAccessKey: S3_STORAGE_CONFIG_DEFAULTS.secretAccessKey, + sslEnabled: S3_STORAGE_CONFIG_DEFAULTS.sslEnabled, + forcePathStyle: S3_STORAGE_CONFIG_DEFAULTS.forcePathStyle, + region: S3_STORAGE_CONFIG_DEFAULTS.region, + batchSize: S3_STORAGE_CONFIG_DEFAULTS.delete.batchSize, +} as const satisfies S3StorageConfig; + +export function createS3StorageConfig(overrides: Partial = {}): S3StorageConfig { + return { ...S3_VALIDATED_CONFIG_DEFAULTS, ...overrides }; +} + // ─── FS Storage Config (FsStorageProvider) ─────────────────────────────────── export const FS_STORAGE_CONFIG_DEFAULTS = { @@ -147,6 +162,18 @@ export function createMockFsConfig(overrides: Record = {}): Con } as unknown as ConfigType; } +// ─── FS Validated Storage Config ─────────────────────────────────── + +export const FS_VALIDATED_CONFIG_DEFAULTS = { + basePath: FS_STORAGE_CONFIG_DEFAULTS.basePath, + subPaths: Object.values(FS_STORAGE_CONFIG_DEFAULTS.subPaths), + batchSize: FS_STORAGE_CONFIG_DEFAULTS.delete.batchSize, +} as const satisfies FsStorageConfig; + +export function createFsStorageConfig(overrides: Partial = {}): FsStorageConfig { + return { ...FS_VALIDATED_CONFIG_DEFAULTS, ...overrides }; +} + // ─── JobTrackerClient ───────────────────────────────────────────────────────── export function createMockJobTrackerClient(): JobTrackerClient { diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index 3f07c4b..09603c6 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -1,13 +1,11 @@ -import { accessSync, Stats, statSync } from 'node:fs'; +import type { Stats } from 'node:fs'; import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import type { Logger } from '@map-colonies/js-logger'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { faker } from '@faker-js/faker'; -import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; -import { FsStorageProvider, type FsConfig } from '@src/cleaner/storageProviders/fsStorageProvider'; -import type { ConfigType } from '@src/common/config'; -import { createMockFsConfig, createMockLogger, FS_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; +import { UnrecoverableError } from '@src/cleaner/errors'; +import { FsStorageProvider } from '@src/cleaner/storageProviders/fsStorageProvider'; +import { createFsStorageConfig, createMockLogger, FS_VALIDATED_CONFIG_DEFAULTS } from '../helpers/mocks'; vi.mock('node:fs/promises', () => ({ stat: vi.fn(), @@ -16,20 +14,11 @@ vi.mock('node:fs/promises', () => ({ rm: vi.fn(), })); -vi.mock(import('node:fs'), async (importOriginal) => { - const originModule = await importOriginal(); - return { - ...originModule, - accessSync: vi.fn(), - statSync: vi.fn(), - }; -}); -const BASE_PATH = FS_STORAGE_CONFIG_DEFAULTS.basePath; +const BASE_PATH = FS_VALIDATED_CONFIG_DEFAULTS.basePath; describe('FsStorageProvider', () => { let provider: FsStorageProvider; let mockLogger: Logger; - let mockConfig: ConfigType; beforeEach(() => { vi.clearAllMocks(); @@ -37,11 +26,16 @@ describe('FsStorageProvider', () => { vi.mocked(unlink).mockResolvedValue(undefined); vi.mocked(rmdir).mockResolvedValue(undefined); vi.mocked(rm).mockResolvedValue(undefined); - vi.mocked(accessSync).mockReturnValue(undefined); - vi.mocked(statSync).mockReturnValue({ isDirectory: () => true } as Stats); mockLogger = createMockLogger(); - mockConfig = createMockFsConfig(); - provider = new FsStorageProvider(mockConfig, mockLogger); + provider = new FsStorageProvider(createFsStorageConfig(), mockLogger); + }); + + describe('#constructor', () => { + it('should return an instance of the class', () => { + const provider = new FsStorageProvider(createFsStorageConfig({ basePath: '/other/base' }), mockLogger); + + expect(provider).toBeInstanceOf(FsStorageProvider); + }); }); describe('#targetExists', () => { @@ -133,6 +127,55 @@ describe('FsStorageProvider', () => { expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); + it('should tag failures with the stringified value when a non-Error is thrown', async () => { + vi.mocked(unlink).mockRejectedValue('raw string failure'); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['raw string failure', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should fall back to "Unknown" when a non-Error empty value is thrown', async () => { + vi.mocked(unlink).mockRejectedValue(''); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should fall back to a generic reason when the thrown value cannot be stringified', async () => { + vi.mocked(unlink).mockRejectedValue({ + toString: () => { + throw new Error('toString failed'); + }, + }); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['non-serializable thrown value', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should unlink every path of a large input', async () => { + const paths = Array.from({ length: 7 }, (_, i) => `tile/10/0/${i}.png`); + + await provider.delete(paths, BASE_PATH); + + expect(unlink).toHaveBeenCalledTimes(7); + for (const path of paths) { + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, path)); + } + }); + + it('should aggregate failures of the same reason keeping the first sample', async () => { + const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + vi.mocked(unlink).mockRejectedValue(permError); + const paths = Array.from({ length: 7 }, (_, i) => `tile/10/0/${i}.png`); + + const result = await provider.delete(paths, BASE_PATH); + + expect(result).toEqual({ failures: new Map([['EACCES', { count: 7, sample: 'tile/10/0/0.png' }]]) }); + }); + it('should handle mixed success, ENOENT and real errors', async () => { const enoent = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); @@ -204,13 +247,52 @@ describe('FsStorageProvider', () => { await provider.delete([], BASE_PATH); expect(rmdir).not.toHaveBeenCalled(); }); + + it('should not call rmdir for a path that has no directory segments', async () => { + await provider.delete(['0.png'], BASE_PATH); + + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, '0.png')); + expect(rmdir).not.toHaveBeenCalled(); + }); + + it('should attempt to rmdir ancestors of paths from every batch', async () => { + // batchSize is 3 → cleanup runs once for all paths, after the last batch + const paths = Array.from({ length: 4 }, (_, i) => `tile/10/${i}/0.png`); + + await provider.delete(paths, BASE_PATH); + + for (let i = 0; i < 4; i++) { + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, `tile/10/${i}`)); + } + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'tile/10')); + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'tile')); + }); + + it('should attempt to rmdir deeper directories before their ancestors', async () => { + await provider.delete(['layer/v1/10/0/0.png'], BASE_PATH); + + const order = vi.mocked(rmdir).mock.calls.map(([path]) => path); + expect(order).toEqual([ + join(BASE_PATH, 'layer/v1/10/0'), + join(BASE_PATH, 'layer/v1/10'), + join(BASE_PATH, 'layer/v1'), + join(BASE_PATH, 'layer'), + ]); + }); }); }); describe('#deleteResources', () => { - const FS_SUB_PATH = FS_STORAGE_CONFIG_DEFAULTS.subPaths.tiles; + const FS_SUB_PATH = FS_VALIDATED_CONFIG_DEFAULTS.subPaths[0]!; const RELATIVE_PATH = 'layer/v1'; + it('should successfully return without failures for empty paths', async () => { + const result = await provider.deleteResources({ paths: [], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).not.toHaveBeenCalled(); + }); + it('should successfully call delete all files and return without failures', async () => { const result = await provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); @@ -277,69 +359,76 @@ describe('FsStorageProvider', () => { await expect(result).resolves.not.toThrow(); }); - }); - describe('#constructor', () => { - it('should construct successfully when the base path is accessible and is a directory', () => { - expect(() => new FsStorageProvider(mockConfig, mockLogger)).not.toThrow(); - expect(accessSync).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath, expect.any(Number)); - expect(statSync).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath); - }); + it('should rm every path when the input spans multiple batches', async () => { + // batchSize is 3 → 7 paths span 3 batches (3 + 3 + 1) + const paths = Array.from({ length: 7 }, (_, i) => `layer/v${i}`); - it('should throw ConfigurationError when the base path does not exist', () => { - vi.mocked(accessSync).mockImplementationOnce(() => { - throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); - }); + await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); - expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + expect(rm).toHaveBeenCalledTimes(7); + for (const path of paths) { + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, path), { recursive: true, force: true }); + } }); - it('should throw ConfigurationError when access to the base path is denied (EACCES)', () => { - vi.mocked(accessSync).mockImplementationOnce(() => { - throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); - }); + it('should accumulate failures of the same reason across batches keeping the first sample', async () => { + vi.mocked(rm).mockRejectedValue(Object.assign(new Error('EACCES'), { code: 'EACCES' })); + const paths = Array.from({ length: 7 }, (_, i) => `layer/v${i}`); - expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); - }); + const result = await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); - it('should throw ConfigurationError when access to the base path is denied (EPERM)', () => { - vi.mocked(accessSync).mockImplementationOnce(() => { - throw Object.assign(new Error('EPERM'), { code: 'EPERM' }); + expect(result).toEqual({ + failures: new Map([['EACCES', { count: 7, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v0') }]]), }); + }); + + it('should group failures by reason across batches', async () => { + const paths = ['layer/v0', 'layer/v1', 'layer/v2', 'layer/v3']; + vi.mocked(rm) + .mockRejectedValueOnce(Object.assign(new Error('EACCES'), { code: 'EACCES' })) + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(Object.assign(new Error('EBUSY'), { code: 'EBUSY' })) + // 4th path lands in the second batch + .mockRejectedValueOnce(Object.assign(new Error('EACCES'), { code: 'EACCES' })); - expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + const result = await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ + failures: new Map([ + ['EACCES', { count: 2, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v0') }], + ['EBUSY', { count: 1, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v2') }], + ]), + }); }); - it('should throw ConfigurationError when the base path exists but is not a directory', () => { - vi.mocked(statSync).mockReturnValueOnce({ isDirectory: () => false } as Stats); + it('should throw UnrecoverableError when a path does not sit under any configured subPath', async () => { + const result = provider.deleteResources({ paths: [RELATIVE_PATH], subPath: 'unconfigured/subPath', storageProvider: 'FS' }); - expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); }); - it('should resolve a relative base path (without a leading separator) to an absolute path', () => { - const relativeConfig = { - get: vi.fn().mockReturnValue({ - ...FS_STORAGE_CONFIG_DEFAULTS, - ...({ basePath: 'relative/tiles' } satisfies Pick), - }), - } as unknown as ConfigType; + it('should throw UnrecoverableError when a path resolves to the configured subPath itself', async () => { + const result = provider.deleteResources({ paths: [''], subPath: FS_SUB_PATH, storageProvider: 'FS' }); - expect(() => new FsStorageProvider(relativeConfig, createMockLogger())).not.toThrow(); - expect(accessSync).toHaveBeenCalledWith('/relative/tiles', expect.any(Number)); + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); }); - it('should throw ConfigurationError for an unexpected accessibility error', () => { - vi.mocked(accessSync).mockImplementationOnce(() => { - throw new Error('disk exploded'); - }); + it('should throw UnrecoverableError when a path escapes into a sibling that shares the subPath prefix', async () => { + // resolves to '/artifacts/tiles-backup' — under the base path, but outside the configured subPath + const result = provider.deleteResources({ paths: ['../tiles-backup'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); - expect(() => new FsStorageProvider(mockConfig, mockLogger)).toThrow(ConfigurationError); + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); }); - it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { - const zeroBatchConfig = createMockFsConfig({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + it('should throw UnrecoverableError when only one of several paths escapes the configured subPath', async () => { + const result = provider.deleteResources({ paths: [RELATIVE_PATH, '../tiles-backup'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); - expect(() => new FsStorageProvider(zeroBatchConfig, mockLogger)).toThrow(ConfigurationError); + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); }); }); }); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 392cc6a..50c9cd7 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -12,10 +12,9 @@ import { import { faker } from '@faker-js/faker'; import type { Logger } from '@map-colonies/js-logger'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { ConfigurationError, UnrecoverableError } from '@src/cleaner/errors'; +import { UnrecoverableError } from '@src/cleaner/errors'; import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; -import type { ConfigType } from '@src/common/config'; -import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; +import { createMockLogger, createS3StorageConfig, S3_VALIDATED_CONFIG_DEFAULTS } from '../helpers/mocks'; const mockSend = vi.fn(); const mockPaginateListObjectsV2Next = vi.fn(); @@ -45,14 +44,31 @@ const BUCKET = 'test-bucket'; describe('S3StorageProvider', () => { let provider: S3StorageProvider; - let mockConfig: ConfigType; let mockLogger: Logger; beforeEach(() => { vi.clearAllMocks(); - mockConfig = createMockS3Config(); mockLogger = createMockLogger(); - provider = new S3StorageProvider(mockConfig, mockLogger); + provider = new S3StorageProvider(createS3StorageConfig(), mockLogger); + }); + + describe('#constructor', () => { + it('should construct S3Client with config values', () => { + const provider = new S3StorageProvider(createS3StorageConfig(), mockLogger); + expect(S3Client).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: { + accessKeyId: S3_VALIDATED_CONFIG_DEFAULTS.accessKeyId, + secretAccessKey: S3_VALIDATED_CONFIG_DEFAULTS.secretAccessKey, + }, + endpoint: S3_VALIDATED_CONFIG_DEFAULTS.endpoint, + forcePathStyle: S3_VALIDATED_CONFIG_DEFAULTS.forcePathStyle, + region: S3_VALIDATED_CONFIG_DEFAULTS.region, + tls: S3_VALIDATED_CONFIG_DEFAULTS.sslEnabled, + }) + ); + expect(provider).toBeInstanceOf(S3StorageProvider); + }); }); describe('#delete', () => { @@ -149,14 +165,10 @@ describe('S3StorageProvider', () => { }); }); - it('should batch paths into chunks of 1000 (S3 limit)', async () => { + it('should batch paths into chunks of the configured batch size', async () => { const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); - const mockConfig = createMockS3Config({ - delete: { - batchSize: 1000, - }, - }); - provider = new S3StorageProvider(mockConfig, mockLogger); + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 1000 }), mockLogger); + await provider.delete(paths, BUCKET); expect(mockSend).toHaveBeenCalledTimes(2); @@ -170,14 +182,9 @@ describe('S3StorageProvider', () => { expect(secondCallInput.Delete.Objects).toHaveLength(500); }); - it('should cap DeleteObjects requests at S3 max keys limit (1000 keys) even when configured batchSize is larger', async () => { + it('should never exceed the S3 max keys limit per request for the maximum allowed batch size', async () => { const paths = Array.from({ length: 2500 }, (_, i) => `object-${i}.txt`); - const mockConfig = createMockS3Config({ - delete: { - batchSize: 2000, - }, - }); - provider = new S3StorageProvider(mockConfig, mockLogger); + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 1000 }), mockLogger); await provider.delete(paths, BUCKET); @@ -198,6 +205,32 @@ describe('S3StorageProvider', () => { expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'object-0.txt' }]]) }); }); + it('should ignore returned errors that carry no Key', async () => { + mockSend.mockResolvedValue({ + Errors: [{ Code: 'InternalError' }, { Key: 'b.txt', Code: 'AccessDenied' }], + }); + + const result = await provider.delete(['a.txt', 'b.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'b.txt' }]]) }); + }); + + it('should return an empty failures map when every returned error carries no Key', async () => { + mockSend.mockResolvedValue({ Errors: [{ Code: 'InternalError' }] }); + + const result = await provider.delete(['a.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map() }); + }); + + it('should tag the chunk with the stringified value when send rejects with a non-Error', async () => { + mockSend.mockRejectedValue('connection reset'); + + const result = await provider.delete(['a.txt', 'b.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map([['connection reset', { count: 2, sample: 'a.txt' }]]) }); + }); + it('should add entire chunk to failures tagged with the thrown error when send rejects', async () => { mockSend.mockRejectedValue(new Error('Network error')); const paths = ['a.txt', 'b.txt']; @@ -267,6 +300,16 @@ describe('S3StorageProvider', () => { const PATH = 'layer/v1'; const NORMALIZED_PATH = `${PATH}/`; + it('should return empty result when input paths is empty', async () => { + const result = await provider.deleteResources({ paths: [], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + it('should return empty result when listing returns no keys', async () => { mockSend .mockResolvedValueOnce(undefined) // bucket exists @@ -483,6 +526,84 @@ describe('S3StorageProvider', () => { expect(mockSend).toHaveBeenCalledTimes(2); }); + it('should handle a page that reports its KeyCount', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { KeyCount: keys.length, Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(DeleteObjectsCommand).toHaveBeenCalledWith({ Bucket: BUCKET, Delete: { Objects: keys.map((Key) => ({ Key })) } }); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should skip an entire page whose keys all belong to a sibling sharing the prefix', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v10/0/0.png' }, { Key: 'layer/v1x/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(DeleteObjectsCommand).not.toHaveBeenCalled(); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should accumulate failures of the same reason across pages keeping the first sample', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/1.png', Code: 'AccessDenied' }] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/0.png' }] } }) + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/1.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'layer/v1/0/0.png' }]]) }); + }); + + it('should accumulate failures of the same reason across paths keeping the first sample', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for path 1 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete path 1 page 1 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for path 2 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v2/0/0.png', Code: 'AccessDenied' }] }); // delete path 2 page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }) + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v2/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: ['layer/v1', 'layer/v2'], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'layer/v1/0/0.png' }]]) }); + }); + + it('should request pages sized by the configured batch size', async () => { + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 500 }), mockLogger); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ pageSize: 500 }), expect.anything()); + }); + it('should handle empty delete page Errors response', async () => { const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; mockSend @@ -681,23 +802,4 @@ describe('S3StorageProvider', () => { expect(mockSend).toHaveBeenCalledTimes(3); }); }); - - describe('#constructor', () => { - it('should construct S3Client with config values', () => { - expect(S3Client).toHaveBeenCalledWith( - expect.objectContaining({ - endpoint: S3_STORAGE_CONFIG_DEFAULTS.endpoint, - forcePathStyle: S3_STORAGE_CONFIG_DEFAULTS.forcePathStyle, - region: S3_STORAGE_CONFIG_DEFAULTS.region, - tls: S3_STORAGE_CONFIG_DEFAULTS.sslEnabled, - }) - ); - }); - - it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { - const zeroBatchConfig = createMockS3Config({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); - - expect(() => new S3StorageProvider(zeroBatchConfig, mockLogger)).toThrow(ConfigurationError); - }); - }); }); diff --git a/tests/storageProviders/storageConfig.spec.ts b/tests/storageProviders/storageConfig.spec.ts new file mode 100644 index 0000000..a427b7e --- /dev/null +++ b/tests/storageProviders/storageConfig.spec.ts @@ -0,0 +1,136 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { faker } from '@faker-js/faker'; +import { ConfigurationError } from '@src/cleaner/errors'; +import { buildFsStorageConfig, buildS3StorageConfig, type FsConfig } from '@src/cleaner/storageProviders/storageConfig'; +import { assertCanDeleteFromFolder } from '@src/cleaner/utils/fs'; +import type { ConfigType } from '@src/common/config'; +import { + createFsStorageConfig, + createMockFsConfig, + createMockLogger, + createMockS3Config, + createS3StorageConfig, + FS_STORAGE_CONFIG_DEFAULTS, +} from '../helpers/mocks'; + +vi.mock('@src/cleaner/utils/fs', () => ({ + assertCanDeleteFromFolder: vi.fn(), +})); + +describe('storageConfig', () => { + let mockLogger: Logger; + + beforeEach(() => { + vi.clearAllMocks(); + mockLogger = createMockLogger(); + }); + + describe('#buildFsStorageConfig', () => { + let mockConfig: ConfigType; + + beforeEach(() => { + mockConfig = createMockFsConfig(); + }); + + it('should return the validated config', () => { + const result = buildFsStorageConfig(mockConfig, mockLogger); + + expect(result).toEqual(createFsStorageConfig()); + }); + + it('should assert the resolved base path is deletable from', () => { + buildFsStorageConfig(mockConfig, mockLogger); + + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath, mockLogger); + }); + + it('should flatten every configured subPath into the returned list', () => { + const config = createMockFsConfig({ subPaths: { tiles: 'artifacts/tiles', gpkg: 'artifacts/gpkg' } satisfies FsConfig['subPaths'] }); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.subPaths).toEqual(['artifacts/tiles', 'artifacts/gpkg']); + }); + + it('should resolve a relative base path (without a leading separator) to an absolute path', () => { + const config = createMockFsConfig({ basePath: 'relative/tiles' } satisfies Pick); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.basePath).toBe('/relative/tiles'); + // the assertion must run against the resolved path, not the raw configured one + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith('/relative/tiles', mockLogger); + }); + + it('should normalize a base path containing traversal segments before asserting it', () => { + const config = createMockFsConfig({ basePath: '/test/tiles/../tiles' } satisfies Pick); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.basePath).toBe('/test/tiles'); + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith('/test/tiles', mockLogger); + }); + + it('should propagate a ConfigurationError raised by the base path assertion', () => { + const expectedError = new ConfigurationError('FS path does not exist: /test'); + vi.mocked(assertCanDeleteFromFolder).mockImplementation(() => { + throw expectedError; + }); + + expect(() => buildFsStorageConfig(mockConfig, mockLogger)).toThrow(expectedError); + }); + + it('should throw ConfigurationError when no subPaths are configured', () => { + const config = createMockFsConfig({ subPaths: {} satisfies FsConfig['subPaths'] }); + + expect(() => buildFsStorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const config = createMockFsConfig({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => buildFsStorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + }); + + describe('#buildS3StorageConfig', () => { + it('should return the validated config', () => { + const config = createMockS3Config(); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result).toStrictEqual(createS3StorageConfig()); + }); + + it('should keep a configured batchSize that is below the S3 max keys limit', () => { + const config = createMockS3Config({ delete: { batchSize: 250 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(250); + }); + + it('should clamp a configured batchSize above the S3 max keys limit to 1000', () => { + const config = createMockS3Config({ delete: { batchSize: 2000 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(1000); + }); + + it('should keep a configured batchSize that is exactly the S3 max keys limit', () => { + const config = createMockS3Config({ delete: { batchSize: 1000 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(1000); + }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const config = createMockS3Config({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => buildS3StorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + }); +}); diff --git a/tests/strategies/deleteStoredResourcesStrategy.spec.ts b/tests/strategies/deleteStoredResourcesStrategy.spec.ts index 348baa6..1ff0a63 100644 --- a/tests/strategies/deleteStoredResourcesStrategy.spec.ts +++ b/tests/strategies/deleteStoredResourcesStrategy.spec.ts @@ -125,6 +125,35 @@ describe('DeleteStoredResourcesStrategy', () => { await expect(result).rejects.toThrow(RecoverableError); }); + it('should include the failures count, grouped reasons and samples in the RecoverableError message', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ + failures: new Map([ + ['AccessDenied', { count: 1, sample: 'layer1/0/0.png' }], + ['InternalError', { count: 4, sample: 'layer1/0/1.png' }], + ]), + }); + + // reasons and samples are ordered by descending count + await expect(strategy.execute(s3Params)).rejects.toThrow( + 'Failed to delete 5 objects. Reasons: InternalError=4, AccessDenied=1. Samples: layer1/0/1.png (InternalError), layer1/0/0.png (AccessDenied)' + ); + }); + + it('should pass every path through to the provider', async () => { + const params: DeleteStoredResourcesParams = { ...s3Params, paths: ['layer1', 'layer2', 'layer3'] }; + + await strategy.execute(params); + + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: params.paths, bucket: S3_BUCKET, storageProvider: 'S3' }); + }); + + it('should resolve without throwing for an empty paths list', async () => { + const result = strategy.execute({ ...s3Params, paths: [] }); + + await expect(result).resolves.toBeUndefined(); + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: [], bucket: S3_BUCKET, storageProvider: 'S3' }); + }); + it('should rethrow error thrown by deleteResources', async () => { const expectedError = new Error('Custom'); (mockS3Provider.deleteResources as ReturnType).mockRejectedValueOnce(expectedError); diff --git a/tests/strategies/tilesDeletionStrategy.spec.ts b/tests/strategies/tilesDeletionStrategy.spec.ts index 31e7c69..d6dfc10 100644 --- a/tests/strategies/tilesDeletionStrategy.spec.ts +++ b/tests/strategies/tilesDeletionStrategy.spec.ts @@ -47,7 +47,7 @@ describe('TilesDeletionStrategy', () => { strategy = new TilesDeletionStrategy(createMockLogger(), createMockStrategyConfig(), storageProviders, queueClient, TASK_CONTEXT); }); - describe('validate', () => { + describe('#validate', () => { it('should validate and return S3 params', () => { expect(strategy.validate(s3Params)).toEqual(s3Params); }); @@ -90,7 +90,7 @@ describe('TilesDeletionStrategy', () => { }); }); - describe('execute', () => { + describe('#execute', () => { describe('target validation', () => { it('should throw UnrecoverableError when S3 storage target does not exist', async () => { vi.mocked(MockS3Provider.targetExists).mockResolvedValue(false); @@ -117,6 +117,14 @@ describe('TilesDeletionStrategy', () => { expect(MockFsProvider.targetExists).toHaveBeenCalledWith(join(FS_BASE_PATH, FS_SUB_PATH), fsParams.tilesPath); }); + + it('should propagate an error thrown by the target existence check', async () => { + const expectedError = new Error('EACCES'); + vi.mocked(MockS3Provider.targetExists).mockRejectedValue(expectedError); + + await expect(strategy.execute(s3Params)).rejects.toThrow(expectedError); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); + }); }); describe('provider routing', () => { @@ -220,6 +228,52 @@ describe('TilesDeletionStrategy', () => { expect(mockUpdateProgress).not.toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); }); + it('should report the percentage of tiles processed so far', async () => { + // batchSize=100, concurrency=2 → flush + progress report after the first 200 of 210 tiles + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 13, minY: 0, maxY: 14 }], + }; + + await strategy.execute(params); + + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, Math.round((200 / 210) * 100)); + }); + + it('should report progress once per completed concurrency window', async () => { + // 420 tiles → two full windows of 200, then a trailing batch of 20 + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 20, minY: 0, maxY: 19 }], + }; + + await strategy.execute(params); + + expect(mockUpdateProgress).toHaveBeenCalledTimes(2); + expect(mockUpdateProgress).toHaveBeenNthCalledWith(1, JOB_ID, TASK_ID, Math.round((200 / 420) * 100)); + expect(mockUpdateProgress).toHaveBeenNthCalledWith(2, JOB_ID, TASK_ID, Math.round((400 / 420) * 100)); + }); + + it('should not report progress for a tile set smaller than one concurrency window', async () => { + await strategy.execute(s3Params); + + expect(mockUpdateProgress).not.toHaveBeenCalled(); + }); + + it('should not flush an empty trailing batch when the tile count divides evenly', async () => { + // 200 tiles = exactly batchSize (100) × concurrency (2) → one window, no remainder + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + + await strategy.execute(params); + + expect(MockS3Provider.delete).toHaveBeenCalledTimes(2); + expect(mockUpdateProgress).toHaveBeenCalledTimes(1); + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); + }); + it('should pass the correct jobId and taskId on mid-stream updates', async () => { const params: TilesDeletionParams = { ...s3Params, @@ -326,6 +380,55 @@ describe('TilesDeletionStrategy', () => { await expect(strategy.execute(s3Params)).rejects.toThrow(/S3 connection lost/); }); + + it('should aggregate hard failures of the same reason across concurrent batches', async () => { + // 200 tiles → two batches of 100, both rejecting with the same error + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete).mockRejectedValue(new Error('S3 connection lost')); + + // count is the sum of both batches, sample comes from the first batch that failed + await expect(strategy.execute(params)).rejects.toThrow( + `Failed to delete 200 tiles. Reasons: S3 connection lost=200. Samples: ${tilePath(5, 0, 0)} (S3 connection lost)` + ); + }); + + it('should aggregate hard failures of different reasons across concurrent batches', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete).mockRejectedValueOnce(new Error('first down')).mockRejectedValueOnce(new Error('second down')); + + await expect(strategy.execute(params)).rejects.toThrow(/Reasons: first down=100, second down=100/); + }); + + it('should surface both soft failures and hard rejections from the same flush', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete) + .mockResolvedValueOnce({ failures: new Map([['AccessDenied', { count: 3, sample: tilePath(5, 0, 0) }]]) }) + .mockRejectedValueOnce(new Error('S3 connection lost')); + + // reasons are ordered by descending count + await expect(strategy.execute(params)).rejects.toThrow(/Failed to delete 103 tiles\. Reasons: S3 connection lost=100, AccessDenied=3/); + }); + + it('should tag hard-rejected batches with the errno code when the thrown error carries one', async () => { + vi.mocked(MockFsProvider.delete).mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' })); + + await expect(strategy.execute(fsParams)).rejects.toThrow(/Reasons: EACCES=4/); + }); + + it('should treat a hard-rejected batch tagged ENOENT as missing tiles rather than a retryable failure', async () => { + vi.mocked(MockFsProvider.delete).mockRejectedValue(Object.assign(new Error('no such file'), { code: 'ENOENT' })); + + await expect(strategy.execute(fsParams)).resolves.toBeUndefined(); + }); }); }); }); diff --git a/tests/utils/fs.spec.ts b/tests/utils/fs.spec.ts new file mode 100644 index 0000000..0a39473 --- /dev/null +++ b/tests/utils/fs.spec.ts @@ -0,0 +1,97 @@ +import { accessSync, constants, statSync, type Stats } from 'node:fs'; +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ConfigurationError } from '@src/cleaner/errors'; +import { assertCanDeleteFromFolder } from '@src/cleaner/utils/fs'; +import { createMockLogger } from '../helpers/mocks'; + +vi.mock(import('node:fs'), async (importOriginal) => { + const originModule = await importOriginal(); + return { + ...originModule, + accessSync: vi.fn(), + statSync: vi.fn(), + }; +}); + +const PATH = '/test/tiles'; + +describe('fs', () => { + describe('#assertCanDeleteFromFolder', () => { + let mockLogger: Logger; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(accessSync).mockReturnValue(undefined); + vi.mocked(statSync).mockReturnValue({ isDirectory: () => true } as Stats); + mockLogger = createMockLogger(); + }); + + it('should not throw when the path is an accessible directory', () => { + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).not.toThrow(); + }); + + it('should check the path for existence, read and write access', () => { + assertCanDeleteFromFolder(PATH, mockLogger); + + expect(accessSync).toHaveBeenCalledWith(PATH, constants.F_OK | constants.R_OK | constants.W_OK); + }); + + it('should check that the path is a directory', () => { + assertCanDeleteFromFolder(PATH, mockLogger); + + expect(statSync).toHaveBeenCalledWith(PATH); + }); + + it('should throw ConfigurationError when the path does not exist (ENOENT)', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow(new ConfigurationError(`FS path does not exist: ${PATH}`)); + }); + + it('should throw ConfigurationError when access is denied (EACCES)', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow(new ConfigurationError(`FS path permission denied for path: ${PATH}`)); + }); + + it('should throw ConfigurationError describing an unexpected accessibility error', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('too many open files'), { code: 'EMFILE' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError('An unexpected error occurred on FS path accessibility check: EMFILE') + ); + }); + + it('should throw ConfigurationError when the path exists but is a file', () => { + vi.mocked(statSync).mockReturnValue({ isDirectory: () => false } satisfies Partial as Stats); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError(`FS path exists but it is a file, not a directory: ${PATH}`) + ); + }); + + it('should throw ConfigurationError when the info check fails unexpectedly', () => { + // e.g. the directory is removed between the accessSync and statSync calls + vi.mocked(statSync).mockImplementation(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError('An unexpected error occurred on FS info check: ENOENT') + ); + }); + + it('should not re-wrap the not-a-directory ConfigurationError as an info check failure', () => { + vi.mocked(statSync).mockReturnValue({ isDirectory: () => false } satisfies Partial as Stats); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).not.toThrow(/An unexpected error occurred on FS info check/); + }); + }); +}); From 479fd27a1b8e844563f985209f525db521d995e3 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:39:21 +0300 Subject: [PATCH 66/66] test: add mergeFailures tests --- .../storageProviders/failuresHandling.spec.ts | 113 ++++++++++++------ 1 file changed, 78 insertions(+), 35 deletions(-) diff --git a/tests/storageProviders/failuresHandling.spec.ts b/tests/storageProviders/failuresHandling.spec.ts index d1863ce..016af25 100644 --- a/tests/storageProviders/failuresHandling.spec.ts +++ b/tests/storageProviders/failuresHandling.spec.ts @@ -1,51 +1,94 @@ import { describe, it, expect } from 'vitest'; -import { summarizeDeleteFailures } from '@src/cleaner/storageProviders'; +import { mergeFailures, summarizeDeleteFailures, type DeleteFailure } from '@src/cleaner/storageProviders'; -describe('#summarizeDeleteFailures', () => { - it('should return zero counts, empty summary for an empty input', () => { - const failures = { failures: new Map() }; +describe('failuresHandling', () => { + describe('#mergeFailures', () => { + it('should return an empty map when both source and target are empty', () => { + expect(mergeFailures({ source: new Map(), target: new Map() })).toEqual(new Map()); + }); - const result = summarizeDeleteFailures(failures); + it('should return the target entries unchanged for an empty source', () => { + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); - expect(result).toEqual({ failuresCount: 0, summary: '', samples: [] }); - }); + expect(mergeFailures({ source: new Map(), target })).toEqual(target); + }); - it('should format the summary with reasons and samples sorted by descending count', () => { - const failures = { - failures: new Map([ - ['EACCES', { count: 1, sample: 'a' }], - ['ENOENT', { count: 3, sample: 'b' }], - ]), - }; + it('should keep entries of both maps when reasons do not overlap', () => { + const source: DeleteFailure = new Map([['EACCES', { count: 1, sample: 'b' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); - const result = summarizeDeleteFailures(failures); + expect(mergeFailures({ source, target })).toEqual( + new Map([ + ['ENOENT', { count: 2, sample: 'a' }], + ['EACCES', { count: 1, sample: 'b' }], + ]) + ); + }); - expect(result).toStrictEqual({ failuresCount: 4, summary: 'ENOENT=3, EACCES=1', samples: ['b (ENOENT)', 'a (EACCES)'] }); - }); + it('should sum counts and keep the target sample when a reason appears in both maps', () => { + const source: DeleteFailure = new Map([['ENOENT', { count: 3, sample: 'source-sample' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'target-sample' }]]); + + expect(mergeFailures({ source, target })).toEqual(new Map([['ENOENT', { count: 5, sample: 'target-sample' }]])); + }); - it('should annotate each sampled path with its reason', () => { - const failures = { - failures: new Map([ - ['ENOENT', { count: 1, sample: 'tile/1.png' }], - ['EACCES', { count: 1, sample: 'tile/2.png' }], - ]), - }; + it('should not mutate the source or the target', () => { + const source: DeleteFailure = new Map([['ENOENT', { count: 3, sample: 'b' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); - const { samples } = summarizeDeleteFailures(failures); + mergeFailures({ source, target }); - expect(samples).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); + expect(target).toEqual(new Map([['ENOENT', { count: 2, sample: 'a' }]])); + expect(source).toEqual(new Map([['ENOENT', { count: 3, sample: 'b' }]])); + }); }); - it('should order samples by descending count', () => { - const failures = { - failures: new Map([ - ['EACCES', { count: 1, sample: 'a' }], - ['ENOENT', { count: 4, sample: 'b' }], - ]), - }; + describe('#summarizeDeleteFailures', () => { + it('should return zero counts, empty summary for an empty input', () => { + const failures = { failures: new Map() }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toEqual({ failuresCount: 0, summary: '', samples: [] }); + }); + + it('should format the summary with reasons and samples sorted by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 3, sample: 'b' }], + ]), + }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toStrictEqual({ failuresCount: 4, summary: 'ENOENT=3, EACCES=1', samples: ['b (ENOENT)', 'a (EACCES)'] }); + }); + + it('should annotate each sampled path with its reason', () => { + const failures = { + failures: new Map([ + ['ENOENT', { count: 1, sample: 'tile/1.png' }], + ['EACCES', { count: 1, sample: 'tile/2.png' }], + ]), + }; + + const { samples } = summarizeDeleteFailures(failures); + + expect(samples).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); + }); + + it('should order samples by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 4, sample: 'b' }], + ]), + }; - const { samples } = summarizeDeleteFailures(failures); + const { samples } = summarizeDeleteFailures(failures); - expect(samples).toEqual(['b (ENOENT)', 'a (EACCES)']); + expect(samples).toEqual(['b (ENOENT)', 'a (EACCES)']); + }); }); });