From 244c37d710a273b6a064b8f56823d138fb4972cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Mon, 7 Sep 2026 17:18:49 +0200 Subject: [PATCH 1/2] Allow partialFilterExpression in backbeat index add payload Since BB-773 (backbeat 9.4.1) the lifecycle conductor sends index specs with a partialFilterExpression, rejected as an unknown field by the Joi schema of POST /_/backbeat/index?operation=add. The v2 lifecycle indexes were never created. Allow the field and add route-level tests. Issue: CLDSRV-991 --- lib/routes/routeBackbeat.js | 1 + tests/functional/backbeat/bucketIndexing.js | 36 +++++++ tests/unit/routes/routeBackbeat.js | 114 ++++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 4ee18a4026..2da9e289a5 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1786,6 +1786,7 @@ const indexEntrySchema = joi.object({ }), ) .required(), + partialFilterExpression: joi.object(), }); const indexingSchema = joi.array().items(indexEntrySchema).min(1); diff --git a/tests/functional/backbeat/bucketIndexing.js b/tests/functional/backbeat/bucketIndexing.js index b0bd57d7af..5c0a39cd1e 100644 --- a/tests/functional/backbeat/bucketIndexing.js +++ b/tests/functional/backbeat/bucketIndexing.js @@ -1,4 +1,5 @@ const assert = require('assert'); +const { promisify } = require('util'); const async = require('async'); const { CreateBucketCommand, @@ -113,6 +114,9 @@ const indexRespObject = [ }, ]; +const indexPut = promisify(indexPutRequest); +const indexGet = promisify(indexGetRequest); + const describeIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; const describeIfNotMongo = process.env.S3METADATA !== 'mongodb' ? describe : describe.skip; @@ -232,6 +236,38 @@ describe('Indexing Routes', () => { }, ], done); }); + + it('should successfully add an index with a partialFilterExpression', async () => { + const payload = [ + { + keys: [ + { key: 'value.last-modified', order: 1 }, + { key: '_id', order: 1 }, + ], + name: 'lifecycleLastModifiedPartial', + partialFilterExpression: { 'value.dataStoreName': 'us-east-1' }, + }, + ]; + await indexPut(payload, TEST_BUCKET); + const data = await indexGet(TEST_BUCKET); + const res = JSON.parse(data.body); + assert(res.Indexes.some(index => index.name === 'lifecycleLastModifiedPartial')); + }); + + it('should return error: partialFilterExpression invalid for mongodb', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badPartialIndex', + partialFilterExpression: { _id: { $regex: 'a' } }, + }, + ]; + await assert.rejects(indexPut(payload, TEST_BUCKET), err => { + assert.strictEqual(err.code, 'InternalError'); + assert.strictEqual(err.statusCode, 500); + return true; + }); + }); }); describeIfNotMongo('without mongodb metadata', () => { diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 5b8edc614f..d727c5f4cb 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -1540,3 +1540,117 @@ describe('routeBackbeat authorization', () => { }); }); }); + +describe('routeBackbeat index add payload validation', () => { + const bucketName = 'bucketname'; + let endPromise; + let resolveEnd; + let response; + + function makeIndexRequest(payload) { + const body = JSON.stringify(payload); + return new DummyRequest( + { + method: 'POST', + headers: { 'content-length': body.length }, + url: `/_/backbeat/index/${bucketName}?operation=add`, + }, + body, + ); + } + + beforeEach(() => { + endPromise = new Promise(resolve => { + resolveEnd = resolve; + }); + response = { + setHeader: sinon.stub(), + writeHead: sinon.stub(), + end: sinon.stub().callsFake((body, encoding, callback) => { + resolveEnd(); + if (callback) { + callback(); + } + }), + }; + sinon.stub(auth.server, 'doAuth').yields( + null, + new AuthInfo({ + canonicalID: 'abcdef/lifecycle', + accountDisplayName: 'Lifecycle Service Account', + }), + undefined, + undefined, + {}, + ); + sinon.stub(metadata, 'putBucketIndexes').yields(null); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should accept indexes with a partialFilterExpression', async () => { + const payload = [ + { + keys: [ + { key: 'value.last-modified', order: 1 }, + { key: '_id', order: 1 }, + ], + name: 'partialIndex', + partialFilterExpression: { _id: { $gte: 'a', $lt: 'b' } }, + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + assert.strictEqual(response.writeHead.getCall(0).args[0], 200); + assert.deepStrictEqual(metadata.putBucketIndexes.getCall(0).args[1], payload); + }); + + it('should accept indexes without a partialFilterExpression', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'plainIndex', + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + assert.strictEqual(response.writeHead.getCall(0).args[0], 200); + assert.deepStrictEqual(metadata.putBucketIndexes.getCall(0).args[1], payload); + }); + + it('should reject indexes with unknown fields', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badIndex', + unknownField: true, + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + const err = JSON.parse(response.end.getCall(0).args[0]); + assert.strictEqual(err.code, 'BadRequest'); + assert.strictEqual(metadata.putBucketIndexes.called, false); + }); + + it('should reject a non-object partialFilterExpression', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badIndex', + partialFilterExpression: 'not-an-object', + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + const err = JSON.parse(response.end.getCall(0).args[0]); + assert.strictEqual(err.code, 'BadRequest'); + assert.strictEqual(metadata.putBucketIndexes.called, false); + }); +}); From 8f164d58706c330d4c639666b50285dbc75ee0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Thu, 10 Sep 2026 17:08:48 +0200 Subject: [PATCH 2/2] Reformat with prettier Issue: CLDSRV-991 --- tests/functional/backbeat/bucketIndexing.js | 233 ++++++++++---------- 1 file changed, 120 insertions(+), 113 deletions(-) diff --git a/tests/functional/backbeat/bucketIndexing.js b/tests/functional/backbeat/bucketIndexing.js index 5c0a39cd1e..afff814485 100644 --- a/tests/functional/backbeat/bucketIndexing.js +++ b/tests/functional/backbeat/bucketIndexing.js @@ -1,14 +1,10 @@ const assert = require('assert'); const { promisify } = require('util'); const async = require('async'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../functional/raw-node/utils/makeRequest'); -const BucketUtility = - require('../../functional/aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; @@ -19,57 +15,63 @@ let credentials = null; let backbeatAuthCredentials = null; async function getCredentials() { - const creds = await s3.config.credentials(); - credentials = { - accessKey: creds.accessKeyId, - secretKey: creds.secretAccessKey, - }; + const creds = await s3.config.credentials(); + credentials = { + accessKey: creds.accessKeyId, + secretKey: creds.secretAccessKey, + }; return credentials; } const TEST_BUCKET = 'bucket-for-bucket-indexing'; function indexDeleteRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'delete' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'delete' }, + }, + cb, + ); } function indexPutRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'add' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'add' }, + }, + cb, + ); } function indexGetRequest(bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'GET', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + }, + cb, + ); } const indexReqObject = [ @@ -93,9 +95,7 @@ const indexReqObject = [ const indexRespObject = [ { name: '_id_', - keys: [ - { key: '_id', order: 1 }, - ] + keys: [{ key: '_id', order: 1 }], }, { keys: [ @@ -140,19 +140,21 @@ describe('Indexing Routes', () => { }); it('should reject non-authenticated requests', done => { - makeRequest({ - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - '/_/backbeat/index/testbucket', - headers: {}, - jsonResponse: true, - }, err => { - assert(err); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); + makeRequest( + { + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/index/testbucket', + headers: {}, + jsonResponse: true, + }, + err => { + assert(err); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); }); it('should return error: invalid payload - empty', done => { @@ -181,60 +183,66 @@ describe('Indexing Routes', () => { describeIfMongo('with mongodb metadata', () => { it('should successfully add indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - ], done); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + ], + done, + ); }); it('should successfully delete indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - next => { - indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, [ - { - name: '_id_', - keys: [{ key: '_id', order: 1 }], - } - ]); - next(); - }); - }, - ], done); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + next => { + indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, [ + { + name: '_id_', + keys: [{ key: '_id', order: 1 }], + }, + ]); + next(); + }); + }, + ], + done, + ); }); it('should successfully add an index with a partialFilterExpression', async () => { @@ -299,4 +307,3 @@ describe('Indexing Routes', () => { }); }); }); -