Skip to content

Allow partialFilterExpression in backbeat index add payload #6274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bert-e merged 2 commits into from
Sep 10, 2026
+269 −111
Merged

Allow partialFilterExpression in backbeat index add payload #6274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
244c37d
Allow partialFilterExpression in backbeat index add payload
maeldonn Sep 7, 2026
8f164d5
Reformat with prettier
maeldonn Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/routes/routeBackbeat.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1786,6 +1786,7 @@ const indexEntrySchema = joi.object({
}),
)
.required(),
partialFilterExpression: joi.object(),
Comment thread
maeldonn marked this conversation as resolved.
});

const indexingSchema = joi.array().items(indexEntrySchema).min(1);
Expand Down
265 changes: 154 additions & 111 deletions tests/functional/backbeat/bucketIndexing.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
const assert = require('assert');
const { promisify } = require('util');
const async = require('async');
const {
CreateBucketCommand,
DeleteBucketCommand,
} = require('@aws-sdk/client-s3');
const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3');

const { makeRequest } = require('../../functional/raw-node/utils/makeRequest');
const BucketUtility =
require('../../functional/aws-node-sdk/lib/utility/bucket-util');
const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util');

const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';

Expand All @@ -18,57 +15,63 @@ let credentials = null;
let backbeatAuthCredentials = null;

async function getCredentials() {
const creds = await s3.config.credentials();
credentials = {
accessKey: creds.accessKeyId,
secretKey: creds.secretAccessKey,
};
const creds = await s3.config.credentials();
credentials = {
accessKey: creds.accessKeyId,
secretKey: creds.secretAccessKey,
};
return credentials;
}

const TEST_BUCKET = 'bucket-for-bucket-indexing';

function indexDeleteRequest(payload, bucket, cb) {
makeRequest({
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'POST',
path:
`/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
requestBody: JSON.stringify(payload),
queryObj: { operation: 'delete' },
}, cb);
makeRequest(
{
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'POST',
path: `/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
requestBody: JSON.stringify(payload),
queryObj: { operation: 'delete' },
},
cb,
);
}

function indexPutRequest(payload, bucket, cb) {
makeRequest({
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'POST',
path:
`/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
requestBody: JSON.stringify(payload),
queryObj: { operation: 'add' },
}, cb);
makeRequest(
{
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'POST',
path: `/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
requestBody: JSON.stringify(payload),
queryObj: { operation: 'add' },
},
cb,
);
}

function indexGetRequest(bucket, cb) {
makeRequest({
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'GET',
path:
`/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
}, cb);
makeRequest(
{
authCredentials: backbeatAuthCredentials,
hostname: ipAddress,
port: 8000,
method: 'GET',
path: `/_/backbeat/index/${bucket}`,
headers: {},
jsonResponse: true,
},
cb,
);
}

const indexReqObject = [
Expand All @@ -92,9 +95,7 @@ const indexReqObject = [
const indexRespObject = [
{
name: '_id_',
keys: [
{ key: '_id', order: 1 },
]
keys: [{ key: '_id', order: 1 }],
},
{
keys: [
Expand All @@ -113,6 +114,9 @@ const indexRespObject = [
},
];

const indexPut = promisify(indexPutRequest);
const indexGet = promisify(indexGetRequest);

const describeIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip;
const describeIfNotMongo = process.env.S3METADATA !== 'mongodb' ? describe : describe.skip;

Expand All @@ -136,19 +140,21 @@ describe('Indexing Routes', () => {
});

it('should reject non-authenticated requests', done => {
makeRequest({
hostname: ipAddress,
port: 8000,
method: 'GET',
path:
'/_/backbeat/index/testbucket',
headers: {},
jsonResponse: true,
}, err => {
assert(err);
assert.strictEqual(err.code, 'AccessDenied');
done();
});
makeRequest(
{
hostname: ipAddress,
port: 8000,
method: 'GET',
path: '/_/backbeat/index/testbucket',
headers: {},
jsonResponse: true,
},
err => {
assert(err);
assert.strictEqual(err.code, 'AccessDenied');
done();
},
);
});

it('should return error: invalid payload - empty', done => {
Expand Down Expand Up @@ -177,60 +183,98 @@ describe('Indexing Routes', () => {

describeIfMongo('with mongodb metadata', () => {
it('should successfully add indexes', done => {
async.series([
next => {
indexPutRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, indexRespObject);
next();
});
},
], done);
async.series(
[
next => {
indexPutRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, indexRespObject);
next();
});
},
],
done,
);
});

it('should successfully delete indexes', done => {
async.series([
next => {
indexPutRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, indexRespObject);
next();
});
},
next => {
indexDeleteRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
async.series(
[
next => {
indexPutRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, indexRespObject);
next();
});
},
next => {
indexDeleteRequest(indexReqObject, TEST_BUCKET, err => {
assert.ifError(err);
next();
});
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, [
{
name: '_id_',
keys: [{ key: '_id', order: 1 }],
},
]);
next();
});
},
],
done,
);
});

it('should successfully add an index with a partialFilterExpression', async () => {
const payload = [
{
keys: [
{ key: 'value.last-modified', order: 1 },
{ key: '_id', order: 1 },
],
name: 'lifecycleLastModifiedPartial',
partialFilterExpression: { 'value.dataStoreName': 'us-east-1' },
},
next => {
indexGetRequest(TEST_BUCKET, (err, data) => {
assert.ifError(err);
const res = JSON.parse(data.body);
assert.deepStrictEqual(res.Indexes, [
{
name: '_id_',
keys: [{ key: '_id', order: 1 }],
}
]);
next();
});
];
await indexPut(payload, TEST_BUCKET);
const data = await indexGet(TEST_BUCKET);
const res = JSON.parse(data.body);
assert(res.Indexes.some(index => index.name === 'lifecycleLastModifiedPartial'));
});

it('should return error: partialFilterExpression invalid for mongodb', async () => {
const payload = [
{
keys: [{ key: '_id', order: 1 }],
name: 'badPartialIndex',
partialFilterExpression: { _id: { $regex: 'a' } },
},
], done);
];
await assert.rejects(indexPut(payload, TEST_BUCKET), err => {
assert.strictEqual(err.code, 'InternalError');
assert.strictEqual(err.statusCode, 500);
return true;
});
});
});

Expand Down Expand Up @@ -263,4 +307,3 @@ describe('Indexing Routes', () => {
});
});
});

Loading
Loading