diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 4ee18a4026..2da9e289a5 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1786,6 +1786,7 @@ const indexEntrySchema = joi.object({ }), ) .required(), + partialFilterExpression: joi.object(), }); const indexingSchema = joi.array().items(indexEntrySchema).min(1); diff --git a/tests/functional/backbeat/bucketIndexing.js b/tests/functional/backbeat/bucketIndexing.js index b0bd57d7af..afff814485 100644 --- a/tests/functional/backbeat/bucketIndexing.js +++ b/tests/functional/backbeat/bucketIndexing.js @@ -1,13 +1,10 @@ const assert = require('assert'); +const { promisify } = require('util'); const async = require('async'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../functional/raw-node/utils/makeRequest'); -const BucketUtility = - require('../../functional/aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; @@ -18,57 +15,63 @@ let credentials = null; let backbeatAuthCredentials = null; async function getCredentials() { - const creds = await s3.config.credentials(); - credentials = { - accessKey: creds.accessKeyId, - secretKey: creds.secretAccessKey, - }; + const creds = await s3.config.credentials(); + credentials = { + accessKey: creds.accessKeyId, + secretKey: creds.secretAccessKey, + }; return credentials; } const TEST_BUCKET = 'bucket-for-bucket-indexing'; function indexDeleteRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'delete' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'delete' }, + }, + cb, + ); } function indexPutRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'add' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'add' }, + }, + cb, + ); } function indexGetRequest(bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'GET', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + }, + cb, + ); } const indexReqObject = [ @@ -92,9 +95,7 @@ const indexReqObject = [ const indexRespObject = [ { name: '_id_', - keys: [ - { key: '_id', order: 1 }, - ] + keys: [{ key: '_id', order: 1 }], }, { keys: [ @@ -113,6 +114,9 @@ const indexRespObject = [ }, ]; +const indexPut = promisify(indexPutRequest); +const indexGet = promisify(indexGetRequest); + const describeIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; const describeIfNotMongo = process.env.S3METADATA !== 'mongodb' ? describe : describe.skip; @@ -136,19 +140,21 @@ describe('Indexing Routes', () => { }); it('should reject non-authenticated requests', done => { - makeRequest({ - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - '/_/backbeat/index/testbucket', - headers: {}, - jsonResponse: true, - }, err => { - assert(err); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); + makeRequest( + { + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/index/testbucket', + headers: {}, + jsonResponse: true, + }, + err => { + assert(err); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); }); it('should return error: invalid payload - empty', done => { @@ -177,60 +183,98 @@ describe('Indexing Routes', () => { describeIfMongo('with mongodb metadata', () => { it('should successfully add indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - ], done); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + ], + done, + ); }); it('should successfully delete indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - next => { - indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + next => { + indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, [ + { + name: '_id_', + keys: [{ key: '_id', order: 1 }], + }, + ]); + next(); + }); + }, + ], + done, + ); + }); + + it('should successfully add an index with a partialFilterExpression', async () => { + const payload = [ + { + keys: [ + { key: 'value.last-modified', order: 1 }, + { key: '_id', order: 1 }, + ], + name: 'lifecycleLastModifiedPartial', + partialFilterExpression: { 'value.dataStoreName': 'us-east-1' }, }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, [ - { - name: '_id_', - keys: [{ key: '_id', order: 1 }], - } - ]); - next(); - }); + ]; + await indexPut(payload, TEST_BUCKET); + const data = await indexGet(TEST_BUCKET); + const res = JSON.parse(data.body); + assert(res.Indexes.some(index => index.name === 'lifecycleLastModifiedPartial')); + }); + + it('should return error: partialFilterExpression invalid for mongodb', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badPartialIndex', + partialFilterExpression: { _id: { $regex: 'a' } }, }, - ], done); + ]; + await assert.rejects(indexPut(payload, TEST_BUCKET), err => { + assert.strictEqual(err.code, 'InternalError'); + assert.strictEqual(err.statusCode, 500); + return true; + }); }); }); @@ -263,4 +307,3 @@ describe('Indexing Routes', () => { }); }); }); - diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 5b8edc614f..d727c5f4cb 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -1540,3 +1540,117 @@ describe('routeBackbeat authorization', () => { }); }); }); + +describe('routeBackbeat index add payload validation', () => { + const bucketName = 'bucketname'; + let endPromise; + let resolveEnd; + let response; + + function makeIndexRequest(payload) { + const body = JSON.stringify(payload); + return new DummyRequest( + { + method: 'POST', + headers: { 'content-length': body.length }, + url: `/_/backbeat/index/${bucketName}?operation=add`, + }, + body, + ); + } + + beforeEach(() => { + endPromise = new Promise(resolve => { + resolveEnd = resolve; + }); + response = { + setHeader: sinon.stub(), + writeHead: sinon.stub(), + end: sinon.stub().callsFake((body, encoding, callback) => { + resolveEnd(); + if (callback) { + callback(); + } + }), + }; + sinon.stub(auth.server, 'doAuth').yields( + null, + new AuthInfo({ + canonicalID: 'abcdef/lifecycle', + accountDisplayName: 'Lifecycle Service Account', + }), + undefined, + undefined, + {}, + ); + sinon.stub(metadata, 'putBucketIndexes').yields(null); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should accept indexes with a partialFilterExpression', async () => { + const payload = [ + { + keys: [ + { key: 'value.last-modified', order: 1 }, + { key: '_id', order: 1 }, + ], + name: 'partialIndex', + partialFilterExpression: { _id: { $gte: 'a', $lt: 'b' } }, + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + assert.strictEqual(response.writeHead.getCall(0).args[0], 200); + assert.deepStrictEqual(metadata.putBucketIndexes.getCall(0).args[1], payload); + }); + + it('should accept indexes without a partialFilterExpression', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'plainIndex', + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + assert.strictEqual(response.writeHead.getCall(0).args[0], 200); + assert.deepStrictEqual(metadata.putBucketIndexes.getCall(0).args[1], payload); + }); + + it('should reject indexes with unknown fields', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badIndex', + unknownField: true, + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + const err = JSON.parse(response.end.getCall(0).args[0]); + assert.strictEqual(err.code, 'BadRequest'); + assert.strictEqual(metadata.putBucketIndexes.called, false); + }); + + it('should reject a non-object partialFilterExpression', async () => { + const payload = [ + { + keys: [{ key: '_id', order: 1 }], + name: 'badIndex', + partialFilterExpression: 'not-an-object', + }, + ]; + routeBackbeat('127.0.0.1', makeIndexRequest(payload), response, log); + void (await endPromise); + + const err = JSON.parse(response.end.getCall(0).args[0]); + assert.strictEqual(err.code, 'BadRequest'); + assert.strictEqual(metadata.putBucketIndexes.called, false); + }); +});