Skip to content

CLDSRV-956: accept RestoreObject during the direct-to-cold archive window #6291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bert-e merged 2 commits into from
Sep 15, 2026
+446 −127
Merged

CLDSRV-956: accept RestoreObject during the direct-to-cold archive window #6291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f50ea3f
Accept RestoreObject on an object not yet archived
francoisferrand Sep 7, 2026
e552011
Reformat with prettier
francoisferrand Sep 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 65 additions & 49 deletions lib/api/apiUtils/object/coldStorage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const { ObjectMDArchive } = require('arsenal').models;
const errors = require('arsenal').errors;
const errorInstances = require('arsenal').errorInstances;
const { config } = require('../../../Config');
const { isColdStorageClass } = require('./storageClass');
const { locationConstraints } = config;

const { scaledMsPerDay } = config.getTimeOptions();
Expand All @@ -17,9 +18,7 @@ const { scaledMsPerDay } = config.getTimeOptions();
* @returns {string|undefined} x-amz-restore
*/
function getAmzRestoreResHeader(objMD) {
if (objMD.archive &&
objMD.archive.restoreRequestedAt &&
!objMD.archive.restoreCompletedAt) {
if (objMD.archive && objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt) {
// Avoid race condition by relying on the `archive` MD of the object
// and return the right header after a RESTORE request.
// eslint-disable-next-line
Expand Down Expand Up @@ -48,7 +47,9 @@ function setArchiveInfoHeaders(objMD) {
}

if (objMD.archive) {
headers['x-amz-scal-archive-info'] = JSON.stringify(objMD.archive.archiveInfo);
if (objMD.archive.archiveInfo) {
headers['x-amz-scal-archive-info'] = JSON.stringify(objMD.archive.archiveInfo);
}

if (objMD.archive.restoreRequestedAt) {
headers['x-amz-scal-restore-requested-at'] = new Date(objMD.archive.restoreRequestedAt).toUTCString();
Expand Down Expand Up @@ -83,11 +84,10 @@ function _validateStartRestore(objectMD, log) {
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
// return InvalidObjectState error if the restored object is expired
// but restore info md of this object has not yet been cleared
log.debug('The restored object already expired.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
log.debug('The restored object already expired.', {
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}

Expand All @@ -96,25 +96,34 @@ function _validateStartRestore(objectMD, log) {
// been reset.
return undefined;
}
if (isColdStorageClass(objectMD['x-amz-storage-class']) && !objectMD.archive?.archiveInfo) {
// The object was written directly to a cold location, and is still awaiting its first
// archive: its data is hot, so there is nothing to recall yet. The request is only
// recorded, and initiated once the archive completes; a repeated request simply updates
// the pending one, instead of being rejected as already in progress.
log.debug('The object is not archived yet, the restore is deferred.', {
archive: objectMD.archive,
method: '_validateStartRestore',
});
return undefined;
}
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
if (!isLocationCold) {
// return InvalidObjectState error if the object is not in cold storage,
// not in cold storage means either location cold flag not exists or cold flag is explicit false
log.debug('The bucket of the object is not in a cold storage location.',
{
isLocationCold,
method: '_validateStartRestore',
});
log.debug('The bucket of the object is not in a cold storage location.', {
isLocationCold,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}
if (objectMD.archive?.restoreRequestedAt) {
// return RestoreAlreadyInProgress error if the object is currently being restored
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
log.debug('The object is currently being restored.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
log.debug('The object is currently being restored.', {
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.RestoreAlreadyInProgress;
}
return undefined;
Expand Down Expand Up @@ -142,21 +151,24 @@ function validatePutVersionId(objMD, versionId, log) {

const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
if (!isLocationCold) {
log.error('The object data is not stored in a cold storage location.',
{
isLocationCold,
dataStoreName: objMD.dataStoreName,
method: 'validatePutVersionId',
});
log.error('The object data is not stored in a cold storage location.', {
isLocationCold,
dataStoreName: objMD.dataStoreName,
method: 'validatePutVersionId',
});
return errors.InvalidObjectState;
}

// make sure object archive restoration is in progress
// NOTE: we do not use putObjectVersion to update the restoration period.
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
log.error('object archive restoration is not in progress',
{ method: 'validatePutVersionId', versionId });
if (
!objMD.archive ||
!objMD.archive.restoreRequestedAt ||
!objMD.archive.restoreRequestedDays ||
objMD.archive.restoreCompletedAt ||
objMD.archive.restoreWillExpireAt
) {
log.error('object archive restoration is not in progress', { method: 'validatePutVersionId', versionId });
return errors.InvalidObjectState;
}

Expand All @@ -180,11 +192,11 @@ function _updateObjectExpirationDate(objectMD, log) {
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
log.debug('The restore status of the object.', {
isObjectAlreadyRestored,
method: 'isObjectAlreadyRestored'
method: 'isObjectAlreadyRestored',
});
if (isObjectAlreadyRestored) {
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
expiryDate.setTime(expiryDate.getTime() + objectMD.archive.restoreRequestedDays * scaledMsPerDay);

/* eslint-disable no-param-reassign */
objectMD.archive.restoreWillExpireAt = expiryDate;
Expand All @@ -208,22 +220,20 @@ function _updateObjectExpirationDate(objectMD, log) {
*
*/
function _updateRestoreInfo(objectMD, restoreParam, log) {
/* eslint-disable no-param-reassign */
if (!objectMD.archive) {
log.debug('objectMD.archive doesn\'t exits', {
objectMD,
method: '_updateRestoreInfo'
});
return errorInstances.InternalError.customizeDescription('Archive metadata is missing.');
// an object awaiting its first archive has no archive metadata yet
objectMD.archive = {};
}
/* eslint-disable no-param-reassign */
objectMD.archive.restoreRequestedAt = new Date();
objectMD.archive.restoreRequestedDays = restoreParam.days;
objectMD.originOp = 's3:ObjectRestore:Post';
/* eslint-enable no-param-reassign */
if (!ObjectMDArchive.isValid(objectMD.archive)) {
// `ObjectMDArchive` requires `archiveInfo`, which is only set once the object is archived
if (objectMD.archive.archiveInfo && !ObjectMDArchive.isValid(objectMD.archive)) {
log.debug('archive is not valid', {
archive: objectMD.archive,
method: '_updateRestoreInfo'
method: '_updateRestoreInfo',
});
return errorInstances.InternalError.customizeDescription('Invalid archive metadata.');
}
Expand All @@ -249,20 +259,20 @@ function startRestore(objectMD, restoreParam, log, cb) {
if (checkResultError) {
log.debug('Restore cannot be done.', {
error: checkResultError,
method: 'startRestore'
method: 'startRestore',
});
return cb(checkResultError);
}
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
if (updateResultError) {
log.debug('Failed to update restore information.', {
error: updateResultError,
method: 'startRestore'
method: 'startRestore',
});
return cb(updateResultError);
}
log.debug('Validated and updated restore information', {
method: 'startRestore'
method: 'startRestore',
});
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
return cb(null, isObjectAlreadyRestored);
Expand All @@ -274,14 +284,20 @@ function startRestore(objectMD, restoreParam, log, cb) {
* @returns {ArsenalError|null} error if object data is not available
*/
function verifyColdObjectAvailable(objMD) {
// An object written directly to a cold location has not been archived yet: its data is still
// in the hot location, and stays available even once a restore has been requested for it.
if (!objMD.archive?.archiveInfo) {
return null;
}
// return error when object is cold
if (objMD.archive &&
// Object is in cold backend
(!objMD.archive.restoreRequestedAt ||
// Object is being restored
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
const err = errorInstances.InvalidObjectState
.customizeDescription('The operation is not valid for the object\'s storage class');
if (
!objMD.archive.restoreRequestedAt ||
// Object is being restored
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt)
) {
const err = errorInstances.InvalidObjectState.customizeDescription(
"The operation is not valid for the object's storage class",
);
return err;
}
return null;
Expand Down
Loading
Loading