diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/kms/hsm/CreateHSMProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/kms/hsm/CreateHSMProfileCmd.java index 496ddee89b27..6ae933a6bde7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/kms/hsm/CreateHSMProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/kms/hsm/CreateHSMProfileCmd.java @@ -87,7 +87,10 @@ public class CreateHSMProfileCmd extends BaseCmd { private String vendorName; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, - description = "HSM configuration details (protocol specific)") + description = "HSM configuration details (protocol specific). For pkcs11, the wrap_algorithm detail " + + "selects how volume encryption keys get wrapped. Allowed values are AES/GCM/NoPadding " + + "(default), AES/KWP/NoPadding (needs Java 18 or later) and AES/CBC/PKCS5Padding. " + + "It can't be changed later.") private Map details; public String getName() { diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java index a7d82d0b9628..5472308ebe91 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java @@ -72,6 +72,12 @@ public VolumeInfo getVolume() { */ AsyncCallFuture expungeVolumeAsync(VolumeInfo volume); + /** + * Releases the encryption key (KMS wrapped key or legacy passphrase) of removed or expunged volumes once nothing + * needs it anymore, e.g. after the last snapshot of such a volume was deleted. + */ + void releaseKeysOfRemovedVolumes(); + void ensureVolumeIsExpungeReady(long volumeId); boolean cloneVolume(long volumeId, long baseVolId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 4cd9a8e23bfc..62d12b75047f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -191,4 +191,27 @@ public interface VolumeDao extends GenericDao, StateDao findByKmsWrappedKeyId(Long kmsWrappedKeyId); + + /** + * All volumes that reference the given KMS wrapped key, including removed ones. Snapshots of a removed volume + * still need the key of that volume. + */ + List listIncludingRemovedByKmsWrappedKeyId(long kmsWrappedKeyId); + + /** + * All volumes that reference the given passphrase, including removed ones. + */ + List listIncludingRemovedByPassphraseId(long passphraseId); + + /** + * Removed or expunged volumes whose KMS wrapped key has not been removed yet. Their key may no longer be needed + * once their snapshots are gone; used by the storage garbage collector to release it. + */ + List listRemovedOrExpungedWithKmsWrappedKey(); + + /** + * Removed or expunged volumes that still reference a passphrase. Used by the storage garbage collector to release + * the passphrase once it is no longer needed. + */ + List listRemovedOrExpungedWithPassphrase(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 1d5ff5e93402..1db1fa40e252 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -23,13 +23,16 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import javax.inject.Inject; +import javax.naming.ConfigurationException; +import org.apache.cloudstack.kms.KMSWrappedKeyVO; +import org.apache.cloudstack.kms.dao.KMSWrappedKeyDao; import org.apache.cloudstack.reservation.ReservationVO; import org.apache.cloudstack.reservation.dao.ReservationDao; -import org.apache.cloudstack.kms.dao.KMSWrappedKeyDao; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -50,6 +53,7 @@ import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -82,6 +86,8 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol private final SearchBuilder poolAndPathSearch; final GenericSearchBuilder CountByOfferingId; private final SearchBuilder kmsMigrationSearch; + private SearchBuilder removedWithPassphraseSearch; + private SearchBuilder removedWithLiveKmsWrappedKeySearch; @Inject ReservationDao reservationDao; @@ -543,6 +549,32 @@ public VolumeDaoImpl() { kmsMigrationSearch.done(); } + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + boolean result = super.configure(name, params); + + removedWithPassphraseSearch = createSearchBuilder(); + removedWithPassphraseSearch.and("passphraseId", removedWithPassphraseSearch.entity().getPassphraseId(), Op.NNULL); + removedWithPassphraseSearch.and().op("removed", removedWithPassphraseSearch.entity().getRemoved(), Op.NNULL); + removedWithPassphraseSearch.or("state", removedWithPassphraseSearch.entity().getState(), Op.EQ); + removedWithPassphraseSearch.cp(); + removedWithPassphraseSearch.done(); + + // Joins with the table of another DAO, so it is built here rather than in the constructor. + SearchBuilder liveWrappedKeySearch = kmsWrappedKeyDao.createSearchBuilder(); + liveWrappedKeySearch.and("wrappedKeyRemoved", liveWrappedKeySearch.entity().getRemoved(), Op.NULL); + + removedWithLiveKmsWrappedKeySearch = createSearchBuilder(); + removedWithLiveKmsWrappedKeySearch.and().op("removed", removedWithLiveKmsWrappedKeySearch.entity().getRemoved(), Op.NNULL); + removedWithLiveKmsWrappedKeySearch.or("state", removedWithLiveKmsWrappedKeySearch.entity().getState(), Op.EQ); + removedWithLiveKmsWrappedKeySearch.cp(); + removedWithLiveKmsWrappedKeySearch.join("wrappedKey", liveWrappedKeySearch, liveWrappedKeySearch.entity().getId(), + removedWithLiveKmsWrappedKeySearch.entity().getKmsWrappedKeyId(), JoinBuilder.JoinType.INNER); + removedWithLiveKmsWrappedKeySearch.done(); + + return result; + } + @Override @DB() public Pair getCountAndTotalByPool(long poolId) { @@ -784,17 +816,6 @@ public boolean remove(Long id) { logger.debug(String.format("Removing volume %s from DB", id)); VolumeVO entry = findById(id); if (entry != null) { - // Clean up KMS wrapped key if volume was encrypted with KMS - if (entry.getKmsWrappedKeyId() != null) { - try { - kmsWrappedKeyDao.remove(entry.getKmsWrappedKeyId()); - logger.debug("Removed KMS wrapped key [id={}] for volume [id={}, uuid={}]", - entry.getKmsWrappedKeyId(), id, entry.getUuid()); - } catch (Exception e) { - logger.warn("Failed to remove KMS wrapped key [id={}] for volume [id={}, uuid={}]: {}", - entry.getKmsWrappedKeyId(), id, entry.getUuid(), e.getMessage(), e); - } - } tagsDao.removeByIdAndType(id, ResourceObjectType.Volume); } boolean result = super.remove(id); @@ -1011,4 +1032,32 @@ public List findByKmsWrappedKeyId(Long kmsWrappedKeyId) { sc.setParameters("notDestroyed", Volume.State.Expunged); return listBy(sc); } + + @Override + public List listIncludingRemovedByKmsWrappedKeyId(long kmsWrappedKeyId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("kmsWrappedKeyId", kmsWrappedKeyId); + return listIncludingRemovedBy(sc); + } + + @Override + public List listIncludingRemovedByPassphraseId(long passphraseId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("passphraseId", passphraseId); + return listIncludingRemovedBy(sc); + } + + @Override + public List listRemovedOrExpungedWithKmsWrappedKey() { + SearchCriteria sc = removedWithLiveKmsWrappedKeySearch.create(); + sc.setParameters("state", Volume.State.Expunged); + return listIncludingRemovedBy(sc); + } + + @Override + public List listRemovedOrExpungedWithPassphrase() { + SearchCriteria sc = removedWithPassphraseSearch.create(); + sc.setParameters("state", Volume.State.Expunged); + return listIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java index ce217cef9e75..c5c6745bda81 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42300to2400.java @@ -16,8 +16,27 @@ // under the License. package com.cloud.upgrade.dao; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.cloud.utils.exception.CloudRuntimeException; + public class Upgrade42300to2400 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + // A removed KMS wrapped key that a volume (not expunged) or a snapshot (not destroyed) still needs + private static final String REMOVED_KMS_WRAPPED_KEY_IN_USE = "w.removed IS NOT NULL AND (" + + "EXISTS (SELECT 1 FROM `cloud`.`volumes` v WHERE v.kms_wrapped_key_id = w.id" + + " AND v.removed IS NULL AND v.state <> 'Expunged')" + + " OR EXISTS (SELECT 1 FROM `cloud`.`snapshots` s JOIN `cloud`.`volumes` v ON v.id = s.volume_id" + + " WHERE v.kms_wrapped_key_id = w.id AND s.removed IS NULL AND s.status <> 'Destroyed'))"; + // The KMS key and the KEK the key was wrapped with still exist. Deleting a KMS key deletes its KEKs but leaves + // its version rows as they are. + private static final String KEK_VERSION_USABLE = "EXISTS (SELECT 1 FROM `cloud`.`kms_kek_versions` k" + + " JOIN `cloud`.`kms_keys` kk ON kk.id = k.kms_key_id" + + " WHERE k.id = w.kek_version_id AND k.removed IS NULL AND k.status <> 'Archived' AND kk.removed IS NULL)"; + @Override public String[] getUpgradableVersionRange() { return new String[]{"4.23.0.0", "24.0.0"}; @@ -27,4 +46,40 @@ public String[] getUpgradableVersionRange() { public String getUpgradedVersion() { return "24.0.0"; } + + @Override + public void performDataMigration(Connection conn) { + restoreKmsWrappedKeysStillInUse(conn); + } + + /** + * In 4.23 removing a volume also removed its KMS wrapped key, even when the key was still needed by the volume + * it was migrated to or by snapshots of the volume. Restores such keys while the KEK they were wrapped with still + * exists, and logs the ones that can't be recovered anymore. + */ + private void restoreKmsWrappedKeysStillInUse(Connection conn) { + String lostKeysQuery = "SELECT w.uuid, v.uuid FROM `cloud`.`kms_wrapped_key` w" + + " JOIN `cloud`.`volumes` v ON v.kms_wrapped_key_id = w.id" + + " WHERE " + REMOVED_KMS_WRAPPED_KEY_IN_USE + " AND NOT " + KEK_VERSION_USABLE; + try (PreparedStatement pstmt = conn.prepareStatement(lostKeysQuery); ResultSet rs = pstmt.executeQuery()) { + while (rs.next()) { + logger.warn("KMS wrapped key [{}] of volume [{}] was removed while still in use and the KEK it was " + + "wrapped with is gone; the volume and its snapshots can't be decrypted anymore", + rs.getString(1), rs.getString(2)); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to look up KMS wrapped keys that were removed while still in use", e); + } + + String restoreQuery = "UPDATE `cloud`.`kms_wrapped_key` w SET w.removed = NULL" + + " WHERE " + REMOVED_KMS_WRAPPED_KEY_IN_USE + " AND " + KEK_VERSION_USABLE; + try (PreparedStatement pstmt = conn.prepareStatement(restoreQuery)) { + int restored = pstmt.executeUpdate(); + if (restored > 0) { + logger.info("Restored {} KMS wrapped key(s) that were removed while still in use", restored); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to restore KMS wrapped keys that were removed while still in use", e); + } + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/KMSWrappedKeyVO.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/KMSWrappedKeyVO.java index 8a763623fba7..724ff49f5e3e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/KMSWrappedKeyVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/KMSWrappedKeyVO.java @@ -61,6 +61,16 @@ public class KMSWrappedKeyVO { @Column(name = "wrapped_blob", nullable = false) private byte[] wrappedBlob; + @Column(name = "wrap_algorithm", nullable = false) + private String wrapAlgorithm; + + /** + * Set on every row that existed when upgrading from 4.23. Such a blob may hold a volume key that was migrated + * from a legacy passphrase in the wrong format; it is checked once and the flag is cleared. + */ + @Column(name = "format_check_pending", nullable = false) + private boolean formatCheckPending; + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) @Temporal(TemporalType.TIMESTAMP) private Date created; @@ -152,6 +162,22 @@ public void setWrappedBlob(byte[] wrappedBlob) { this.wrappedBlob = wrappedBlob != null ? Arrays.copyOf(wrappedBlob, wrappedBlob.length) : null; } + public String getWrapAlgorithm() { + return wrapAlgorithm; + } + + public void setWrapAlgorithm(String wrapAlgorithm) { + this.wrapAlgorithm = wrapAlgorithm; + } + + public boolean isFormatCheckPending() { + return formatCheckPending; + } + + public void setFormatCheckPending(boolean formatCheckPending) { + this.formatCheckPending = formatCheckPending; + } + public Date getCreated() { return created; } @@ -171,6 +197,6 @@ public void setRemoved(Date removed) { @Override public String toString() { return String.format("KMSWrappedKey %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, "id", "uuid", "kmsKeyId", "kekVersionId", "zoneId", "created", "removed")); + this, "id", "uuid", "kmsKeyId", "kekVersionId", "zoneId", "wrapAlgorithm", "created", "removed")); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java index 5e9e418a1667..9bd19c155d20 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDao.java @@ -32,4 +32,10 @@ public interface KMSWrappedKeyDao extends GenericDao { List listByKekVersionId(Long kekVersionId, int limit); long countByKekVersionId(Long kekVersionId); + + /** + * Rows still flagged for the one-time format check after the upgrade from 4.23 + * (see {@link KMSWrappedKeyVO#isFormatCheckPending()}), with an id greater than {@code afterId}, by id. + */ + List listFormatCheckPending(long afterId, int limit); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java index fc1a7190ae8e..076459e01b07 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/kms/dao/KMSWrappedKeyDaoImpl.java @@ -38,6 +38,8 @@ public KMSWrappedKeyDaoImpl() { allFieldSearch.and("kmsKeyId", allFieldSearch.entity().getKmsKeyId(), SearchCriteria.Op.EQ); allFieldSearch.and("kekVersionId", allFieldSearch.entity().getKekVersionId(), SearchCriteria.Op.EQ); allFieldSearch.and("zoneId", allFieldSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + allFieldSearch.and("formatCheckPending", allFieldSearch.entity().isFormatCheckPending(), SearchCriteria.Op.EQ); + allFieldSearch.and("idAfter", allFieldSearch.entity().getId(), SearchCriteria.Op.GT); allFieldSearch.done(); } @@ -67,4 +69,12 @@ public long countByKekVersionId(Long kekVersionId) { Integer count = getCount(sc); return count != null ? count.longValue() : 0L; } + + @Override + public List listFormatCheckPending(long afterId, int limit) { + SearchCriteria sc = allFieldSearch.create(); + sc.setParameters("formatCheckPending", true); + sc.setParameters("idAfter", afterId); + return listBy(sc, new Filter(KMSWrappedKeyVO.class, "id", true, 0L, (long) limit)); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 3365f1f855ba..e470c1a1653a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -239,3 +239,36 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` ( PRIMARY KEY (`id`), CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- KMS: record the algorithm each key was wrapped with. Keys of the PKCS#11 provider were wrapped with +-- AES/CBC/PKCS5Padding (the column default), keys of the database provider with AES/GCM/NoPadding. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kms_wrapped_key', 'wrap_algorithm', 'VARCHAR(64) NOT NULL DEFAULT ''AES/CBC/PKCS5Padding'' COMMENT ''Algorithm the key was wrapped with'''); +UPDATE `cloud`.`kms_wrapped_key` w + JOIN `cloud`.`kms_kek_versions` v ON v.id = w.kek_version_id + JOIN `cloud`.`kms_hsm_profiles` p ON p.id = v.hsm_profile_id +SET w.wrap_algorithm = 'AES/GCM/NoPadding' +WHERE LOWER(p.protocol) = 'database'; + +-- KMS: 4.23 migrated legacy volume passphrases to KMS in a wrong format. Flag every existing key for a one-time +-- check by the management server, which repairs the affected ones. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kms_wrapped_key', 'format_check_pending', 'TINYINT(1) NOT NULL DEFAULT 0 COMMENT ''1 if the key still has to be checked for the format 4.23 migrated legacy passphrases in'''); +UPDATE `cloud`.`kms_wrapped_key` SET format_check_pending = 1; + +-- KMS: existing PKCS#11 HSM profiles keep wrapping keys with AES/CBC/PKCS5Padding. New profiles default to +-- AES/GCM/NoPadding. +INSERT INTO `cloud`.`kms_hsm_profile_details` (profile_id, name, value) + SELECT p.id, 'wrap_algorithm', 'AES/CBC/PKCS5Padding' FROM `cloud`.`kms_hsm_profiles` p + WHERE LOWER(p.protocol) = 'pkcs11' + AND NOT EXISTS (SELECT 1 FROM `cloud`.`kms_hsm_profile_details` d WHERE d.profile_id = p.id AND d.name = 'wrap_algorithm'); +UPDATE `cloud`.`kms_keys` k + JOIN `cloud`.`kms_hsm_profiles` p ON p.id = k.hsm_profile_id +SET k.algorithm = 'AES/CBC/PKCS5Padding' +WHERE LOWER(p.protocol) = 'pkcs11'; + +-- KMS: the database provider doesn't use an HSM +UPDATE `cloud`.`kms_hsm_profiles` SET name = 'Database Provider' WHERE protocol = 'database' AND name = 'HSM Database Provider'; + +-- KMS: remove legacy passphrases no volume refers to anymore, e.g. those left behind when 4.23 migrated a volume to KMS +DELETE p FROM `cloud`.`passphrase` p + LEFT JOIN `cloud`.`volumes` v ON v.passphrase_id = p.id +WHERE v.id IS NULL; diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index de85067aae57..4c08015ab0f9 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -72,9 +72,11 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.exception.CloudRuntimeException; @@ -85,6 +87,7 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -126,6 +129,8 @@ public class VolumeObject implements VolumeInfo { @Inject PassphraseDao passphraseDao; @Inject + SnapshotDao snapshotDao; + @Inject VolumeOrchestrationService orchestrationService; @@ -955,6 +960,8 @@ public void setKmsWrappedKeyId(Long id) { /** * Removes passphrase reference from underlying volume. Also removes the associated passphrase entry if it is the last user. + * A volume that still has snapshots keeps its reference, as restoring or deleting a snapshot needs the passphrase + * of its volume; the storage garbage collector calls this again for such a volume once its snapshots are gone. */ public void deletePassphrase() { Transaction.execute(new TransactionCallbackNoReturn() { @@ -962,13 +969,19 @@ public void deletePassphrase() { public void doInTransactionWithoutResult(TransactionStatus status) { Long passphraseId = volumeVO.getPassphraseId(); if (passphraseId != null) { + if (snapshotDao.listByVolumeId(volumeVO.getId()).stream() + .anyMatch(snapshot -> snapshot.getState() != Snapshot.State.Destroyed)) { + logger.debug("Volume {} still has snapshots, keeping its passphrase", volumeVO); + return; + } volumeVO.setPassphraseId(null); volumeDao.persist(volumeVO); logger.debug("Checking to see if we can delete passphrase id {} for volume {}", passphraseId, volumeVO); - List volumes = volumeDao.listVolumesByPassphraseId(passphraseId); + // Removed volumes count too: they keep their reference while they have snapshots + List volumes = volumeDao.listIncludingRemovedByPassphraseId(passphraseId); - if (volumes != null && !volumes.isEmpty()) { + if (!volumes.isEmpty()) { logger.debug("Other volumes use this passphrase, skipping deletion"); return; } @@ -988,20 +1001,26 @@ public void doInTransactionWithoutResult(TransactionStatus status) { public byte[] getPassphrase() { // First check for KMS-encrypted volume if (volumeVO.getKmsWrappedKeyId() != null) { + byte[] dekBytes; try { // Unwrap the DEK from KMS (returns raw binary bytes) - byte[] dekBytes = kmsManager.unwrapKey(volumeVO.getKmsWrappedKeyId()); - // Base64-encode the DEK for consistency with legacy passphrases - // and for use with qemu-img which expects base64 format - String base64Dek = java.util.Base64.getEncoder().encodeToString(dekBytes); - // Zeroize the raw DEK bytes - java.util.Arrays.fill(dekBytes, (byte) 0); - // Return UTF-8 bytes of the base64 string - return base64Dek.getBytes(java.nio.charset.StandardCharsets.UTF_8); + dekBytes = kmsManager.unwrapKey(volumeVO.getKmsWrappedKeyId()); } catch (KMSException e) { + if (volumeVO.getRemoved() != null && e.getErrorType() == KMSException.ErrorType.KEK_NOT_FOUND) { + // The key of a removed volume can be gone, e.g. released by an older release while snapshots of + // the volume still existed. Deleting such a snapshot doesn't need the key, so don't fail it. + // Other errors (e.g. the HSM is unreachable) must not look like an unencrypted volume. + logger.warn("Failed to unwrap KMS key for removed volume {}: {}", volumeVO, e.getMessage()); + return new byte[0]; + } logger.error("Failed to unwrap KMS key for volume {}: {}", volumeVO, e.getMessage(), e); throw e; } + // Base64-encode the DEK for consistency with legacy passphrases and for use with qemu-img, which expects + // base64 format. Encoding straight to bytes avoids an immutable String copy of the key. + byte[] passphrase = Base64.getEncoder().encode(dekBytes); + Arrays.fill(dekBytes, (byte) 0); + return passphrase; } // Fallback to legacy passphrase-based encryption diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 69b40f64af19..3d1c597c5784 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -391,6 +391,25 @@ private boolean canVolumeBeRemoved(long volumeId) { } + @Override + public void releaseKeysOfRemovedVolumes() { + // Each release keeps the key while another volume or a snapshot still needs it + for (VolumeVO volume : volDao.listRemovedOrExpungedWithKmsWrappedKey()) { + try { + kmsManager.deleteKMSWrappedKey(volume); + } catch (Exception e) { + logger.warn("Failed to release the KMS wrapped key of volume {}: {}", volume, e.getMessage(), e); + } + } + for (VolumeVO volume : volDao.listRemovedOrExpungedWithPassphrase()) { + try { + ((VolumeObject) volFactory.getVolume(volume.getId())).deletePassphrase(); + } catch (Exception e) { + logger.warn("Failed to release the passphrase of volume {}: {}", volume, e.getMessage(), e); + } + } + } + @DB @Override public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java index fc79e32d1143..dcbb3cfd9801 100644 --- a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java @@ -24,14 +24,19 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import java.util.Arrays; +import java.util.Base64; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -43,6 +48,9 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.kms.KMSException; +import org.apache.cloudstack.kms.KMSManager; +import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; @@ -88,6 +96,15 @@ public class VolumeObjectTest extends TestCase{ @Mock VolumeOrchestrationService orchestrationServiceMock; + @Mock + SnapshotDao snapshotDaoMock; + + @Mock + PassphraseDao passphraseDaoMock; + + @Mock + KMSManager kmsManagerMock; + Set> diskOfferingVoMethodsWithLongReturn = new HashSet<>(); List objectInDataStoreStateMachineEvents = Arrays.asList(ObjectInDataStoreStateMachine.Event.values()); @@ -584,4 +601,96 @@ public void validateHandleProcessEventAnswerCopyCmdAnswer(){ Mockito.verify(volumeObjectSpy, Mockito.times(1)).handleProcessEventAnswer(copyCmdAnswer, true, true); } + + @Test + public void deletePassphraseKeepsPassphraseWhileSnapshotsOfTheVolumeExist() { + mockVolumeWithPassphrase(); + SnapshotVO snapshot = Mockito.mock(SnapshotVO.class); + Mockito.doReturn(Snapshot.State.BackedUp).when(snapshot).getState(); + Mockito.doReturn(List.of(snapshot)).when(snapshotDaoMock).listByVolumeId(1L); + + volumeObjectSpy.deletePassphrase(); + + // Restoring or deleting the snapshot needs the passphrase of its volume + Mockito.verify(volumeVoMock, Mockito.never()).setPassphraseId(Mockito.any()); + Mockito.verify(passphraseDaoMock, Mockito.never()).remove(Mockito.anyLong()); + } + + @Test + public void deletePassphraseRemovesPassphraseNoVolumeRefersTo() { + mockVolumeWithPassphrase(); + Mockito.doReturn(List.of()).when(snapshotDaoMock).listByVolumeId(1L); + Mockito.doReturn(List.of()).when(volumeDaoMock).listIncludingRemovedByPassphraseId(7L); + + volumeObjectSpy.deletePassphrase(); + + Mockito.verify(volumeVoMock).setPassphraseId(null); + Mockito.verify(passphraseDaoMock).remove(7L); + } + + @Test + public void deletePassphraseKeepsPassphraseARemovedVolumeStillRefersTo() { + mockVolumeWithPassphrase(); + Mockito.doReturn(List.of()).when(snapshotDaoMock).listByVolumeId(1L); + // e.g. the removed source of a migration, kept for its snapshots + Mockito.doReturn(List.of(Mockito.mock(VolumeVO.class))).when(volumeDaoMock).listIncludingRemovedByPassphraseId(7L); + + volumeObjectSpy.deletePassphrase(); + + Mockito.verify(volumeVoMock).setPassphraseId(null); + Mockito.verify(passphraseDaoMock, Mockito.never()).remove(Mockito.anyLong()); + } + + @Test + public void getPassphraseReturnsBase64OfKmsKey() { + volumeObjectSpy.kmsManager = kmsManagerMock; + Mockito.doReturn(50L).when(volumeVoMock).getKmsWrappedKeyId(); + byte[] key = new byte[32]; + Arrays.fill(key, (byte) 5); + Mockito.doReturn(key.clone()).when(kmsManagerMock).unwrapKey(50L); + + Assert.assertArrayEquals(Base64.getEncoder().encode(key), volumeObjectSpy.getPassphrase()); + } + + @Test + public void getPassphraseReturnsEmptyForRemovedVolumeWhoseKmsKeyIsGone() { + volumeObjectSpy.kmsManager = kmsManagerMock; + Mockito.doReturn(50L).when(volumeVoMock).getKmsWrappedKeyId(); + Mockito.doReturn(new Date()).when(volumeVoMock).getRemoved(); + Mockito.doThrow(KMSException.kekNotFound("Wrapped key not found: 50")).when(kmsManagerMock).unwrapKey(50L); + + // e.g. deleting a snapshot of the volume doesn't need the key + Assert.assertEquals(0, volumeObjectSpy.getPassphrase().length); + } + + @Test(expected = KMSException.class) + public void getPassphraseThrowsForRemovedVolumeWhenHsmIsUnreachable() { + volumeObjectSpy.kmsManager = kmsManagerMock; + Mockito.doReturn(50L).when(volumeVoMock).getKmsWrappedKeyId(); + Mockito.doReturn(new Date()).when(volumeVoMock).getRemoved(); + Mockito.doThrow(new KMSException(KMSException.ErrorType.CONNECTION_FAILED, "HSM unreachable")) + .when(kmsManagerMock).unwrapKey(50L); + + // An empty passphrase would make the volume look unencrypted + volumeObjectSpy.getPassphrase(); + } + + @Test(expected = KMSException.class) + public void getPassphraseThrowsForVolumeWhoseKmsKeyIsGone() { + volumeObjectSpy.kmsManager = kmsManagerMock; + Mockito.doReturn(50L).when(volumeVoMock).getKmsWrappedKeyId(); + Mockito.doThrow(KMSException.kekNotFound("Wrapped key not found: 50")).when(kmsManagerMock).unwrapKey(50L); + + volumeObjectSpy.getPassphrase(); + } + + /** + * Volume 1 using passphrase 7. + */ + private void mockVolumeWithPassphrase() { + volumeObjectSpy.snapshotDao = snapshotDaoMock; + volumeObjectSpy.passphraseDao = passphraseDaoMock; + Mockito.doReturn(1L).when(volumeVoMock).getId(); + Mockito.doReturn(7L).when(volumeVoMock).getPassphraseId(); + } } diff --git a/framework/kms/src/main/java/org/apache/cloudstack/framework/kms/KMSProvider.java b/framework/kms/src/main/java/org/apache/cloudstack/framework/kms/KMSProvider.java index 388d464caa75..be94ff017de5 100644 --- a/framework/kms/src/main/java/org/apache/cloudstack/framework/kms/KMSProvider.java +++ b/framework/kms/src/main/java/org/apache/cloudstack/framework/kms/KMSProvider.java @@ -103,6 +103,19 @@ default String createKek(KeyPurpose purpose, String label, int keyBits) throws K */ void deleteKek(String kekId) throws KMSException; + /** + * Delete a KEK from the secure backend using an explicit HSM profile. Use this when the KEK has no + * version record (yet), so the provider can't find its profile from the KEK identifier. + * WARNING: This will make all DEKs wrapped by this KEK unrecoverable. + * + * @param kekId the KEK identifier to delete + * @param hsmProfileId HSM profile ID holding the KEK (null for auto-resolution/default) + * @throws KMSException if deletion fails or KEK not found + */ + default void deleteKek(String kekId, Long hsmProfileId) throws KMSException { + deleteKek(kekId); + } + /** * Validates the configuration details for this provider before saving an HSM * profile. @@ -124,6 +137,17 @@ default void validateProfileConfig(java.util.Map details) throws */ boolean isKekAvailable(String kekId) throws KMSException; + /** + * Returns the algorithm this provider uses to wrap new keys under the given HSM profile + * (e.g. {@code AES/GCM/NoPadding}). The algorithm is recorded with every wrapped key so it can + * still be unwrapped after the profile or the provider default changes. + * + * @param hsmProfileId optional HSM profile ID (null for auto-resolution/default) + * @return the wrap algorithm name + * @throws KMSException if the profile cannot be resolved or has no valid algorithm configured + */ + String getWrapAlgorithm(Long hsmProfileId) throws KMSException; + /** * Wrap (encrypt) a plaintext Data Encryption Key with a KEK. * Delegates to {@link #wrapKey(byte[], KeyPurpose, String, Long)} with null profile ID. diff --git a/plugins/kms/database/src/main/java/org/apache/cloudstack/kms/provider/DatabaseKMSProvider.java b/plugins/kms/database/src/main/java/org/apache/cloudstack/kms/provider/DatabaseKMSProvider.java index d2dbe3969278..4a09c7541597 100644 --- a/plugins/kms/database/src/main/java/org/apache/cloudstack/kms/provider/DatabaseKMSProvider.java +++ b/plugins/kms/database/src/main/java/org/apache/cloudstack/kms/provider/DatabaseKMSProvider.java @@ -49,9 +49,10 @@ * Database-backed KMS provider that stores master KEKs in a PKCS#11-like object table. * Uses AES-256-GCM for all cryptographic operations. *

- * This provider is suitable for deployments that don't have access to HSM hardware. - * The master KEKs are stored encrypted in the kms_database_kek_objects table using - * CloudStack's existing DBEncryptionUtil, with PKCS#11-compatible attributes. + * This provider is intended for testing and simulator use only. It is not a substitute for an HSM: + * the master KEKs are stored in the CloudStack database (kms_database_kek_objects table), protected + * only by CloudStack's existing DBEncryptionUtil, so anyone with access to the database and the + * management server's database encryption key can recover them. */ public class DatabaseKMSProvider extends AdapterBase implements KMSProvider { private static final Logger logger = LogManager.getLogger(DatabaseKMSProvider.class); @@ -62,7 +63,7 @@ public class DatabaseKMSProvider extends AdapterBase implements KMSProvider { private static final String CKO_SECRET_KEY = "CKO_SECRET_KEY"; private static final String CKK_AES = "CKK_AES"; - private static final String DEFAULT_PROFILE_NAME = "HSM Database Provider"; + private static final String DEFAULT_PROFILE_NAME = "Database Provider"; private static final long SYSTEM_ACCOUNT_ID = 1L; private static final long ROOT_DOMAIN_ID = 1L; @@ -84,6 +85,11 @@ public String getProviderName() { return PROVIDER_NAME; } + @Override + public String getWrapAlgorithm(Long hsmProfileId) { + return ALGORITHM; + } + @Override public String createKek(KeyPurpose purpose, String label, int keyBits, Long hsmProfileId) throws KMSException { // Database provider ignores hsmProfileId diff --git a/plugins/kms/pkcs11/src/main/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProvider.java b/plugins/kms/pkcs11/src/main/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProvider.java index 622a943acd1e..4f9ad2f264d0 100644 --- a/plugins/kms/pkcs11/src/main/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProvider.java +++ b/plugins/kms/pkcs11/src/main/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProvider.java @@ -39,6 +39,7 @@ import javax.crypto.KeyGenerator; import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.IvParameterSpec; import javax.inject.Inject; import java.io.Closeable; @@ -65,6 +66,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -76,14 +78,20 @@ public class PKCS11HSMProvider extends AdapterBase implements KMSProvider { private static final Logger logger = LogManager.getLogger(PKCS11HSMProvider.class); private static final String PROVIDER_NAME = "pkcs11"; - // Security note: AES-CBC provides confidentiality but not authenticity (no - // HMAC). - // While AES-GCM is preferred, SunPKCS11 support for GCM is often buggy or - // missing - // depending on the underlying driver. We rely on the HSM/storage for tamper - // resistance. - // AES-CBC with PKCS5Padding: FIPS-compliant (NIST SP 800-38A) with universal PKCS#11 support - private static final String CIPHER_ALGORITHM = "AES/CBC/PKCS5Padding"; + // Name of the HSM profile detail that selects how DEKs are wrapped under this profile's KEKs. + static final String WRAP_ALGORITHM_DETAIL = "wrap_algorithm"; + // Authenticated: a modified blob fails to unwrap (AES-GCM, NIST SP 800-38D). + static final String ALG_GCM = "AES/GCM/NoPadding"; + // Authenticated key wrap with padding (RFC 5649, NIST SP 800-38F). Needs SunPKCS11 support, i.e. Java 18+. + static final String ALG_KWP = "AES/KWP/NoPadding"; + // Legacy format of 4.23 and earlier. Not authenticated, so a modified blob can unwrap to a different key + // without an error. Kept for HSMs that support neither of the above and to read existing blobs. + static final String ALG_CBC = "AES/CBC/PKCS5Padding"; + private static final Set WRAP_ALGORITHMS = Set.of(ALG_GCM, ALG_KWP, ALG_CBC); + private static final int KWP_MIN_JAVA_FEATURE_VERSION = 18; + private static final int GCM_IV_LENGTH = 12; + private static final int GCM_TAG_BITS = 128; + private static final int AES_BLOCK_LENGTH = 16; private static final long SESSION_ACQUIRE_TIMEOUT_MS = 5000L; @@ -117,8 +125,13 @@ public String createKek(KeyPurpose purpose, String label, int keyBits, Long hsmP @Override public void deleteKek(String kekId) throws KMSException { - Long hsmProfileId = resolveProfileId(kekId); - executeWithSession(hsmProfileId, session -> { + deleteKek(kekId, null); + } + + @Override + public void deleteKek(String kekId, Long hsmProfileId) throws KMSException { + Long profileId = hsmProfileId != null ? hsmProfileId : resolveProfileId(kekId); + executeWithSession(profileId, session -> { session.deleteKey(kekId); return null; }); @@ -140,13 +153,15 @@ Long resolveProfileId(String kekLabel) throws KMSException { * Validates: *

    *
  • {@code library}: Required, should point to PKCS#11 library
    • - *
  • {@code slot}, {@code slot_list_index}, or {@code token_label}: At least - * one required
    • + *
  • {@code slot} or {@code slot_list_index}: At least one required
    • *
  • {@code pin}: Required for HSM authentication
    • *
  • {@code max_sessions}: Optional, must be positive integer if provided
    • + *
  • {@code wrap_algorithm}: Optional, one of {@code AES/GCM/NoPadding} (default, added to + * {@code config} when absent), {@code AES/KWP/NoPadding} (needs Java 18+) or + * {@code AES/CBC/PKCS5Padding} (legacy, unauthenticated)
    • *
* - * @param config Configuration map from HSM profile details + * @param config Configuration map from HSM profile details; the default wrap algorithm is added to it * @throws KMSException with {@code INVALID_PARAMETER} if validation fails */ @Override @@ -158,10 +173,9 @@ public void validateProfileConfig(Map config) throws KMSExceptio String slot = config.get("slot"); String slotListIndex = config.get("slot_list_index"); - String tokenLabel = config.get("token_label"); - if (StringUtils.isAllBlank(slot, slotListIndex, tokenLabel)) { + if (StringUtils.isAllBlank(slot, slotListIndex)) { throw KMSException.invalidParameter( - "One of 'slot', 'slot_list_index', or 'token_label' is required for PKCS#11 HSM profile"); + "One of 'slot' or 'slot_list_index' is required for PKCS#11 HSM profile"); } if (StringUtils.isNotBlank(slot)) { @@ -201,6 +215,17 @@ public void validateProfileConfig(Map config) throws KMSExceptio throw KMSException.invalidParameter("max_sessions must be a valid integer: " + max_sessions); } } + + String wrapAlgorithm = config.get(WRAP_ALGORITHM_DETAIL); + if (StringUtils.isBlank(wrapAlgorithm)) { + config.put(WRAP_ALGORITHM_DETAIL, ALG_GCM); + } else if (!WRAP_ALGORITHMS.contains(wrapAlgorithm)) { + throw KMSException.invalidParameter(String.format("%s must be one of %s: %s", WRAP_ALGORITHM_DETAIL, + WRAP_ALGORITHMS, wrapAlgorithm)); + } else if (ALG_KWP.equals(wrapAlgorithm) && Runtime.version().feature() < KWP_MIN_JAVA_FEATURE_VERSION) { + throw KMSException.invalidParameter(ALG_KWP + " requires Java " + KWP_MIN_JAVA_FEATURE_VERSION + + " or later on the management server"); + } } @Override @@ -213,6 +238,14 @@ public boolean isKekAvailable(String kekId) throws KMSException { } } + @Override + public String getWrapAlgorithm(Long hsmProfileId) throws KMSException { + if (hsmProfileId == null) { + throw KMSException.invalidParameter("HSM Profile ID is required for PKCS#11 provider"); + } + return getSessionPool(hsmProfileId).wrapAlgorithm; + } + @Override public WrappedKey wrapKey(byte[] plainDek, KeyPurpose purpose, String kekLabel, Long hsmProfileId) throws KMSException { @@ -220,8 +253,9 @@ public WrappedKey wrapKey(byte[] plainDek, KeyPurpose purpose, String kekLabel, hsmProfileId = resolveProfileId(kekLabel); } - byte[] wrappedBlob = executeWithSession(hsmProfileId, session -> session.wrapKey(plainDek, kekLabel)); - return new WrappedKey(kekLabel, purpose, CIPHER_ALGORITHM, wrappedBlob, PROVIDER_NAME, new Date(), null); + String algorithm = getWrapAlgorithm(hsmProfileId); + byte[] wrappedBlob = executeWithSession(hsmProfileId, session -> session.wrapKey(plainDek, kekLabel, algorithm)); + return new WrappedKey(kekLabel, purpose, algorithm, wrappedBlob, PROVIDER_NAME, new Date(), null); } @Override @@ -230,15 +264,19 @@ public byte[] unwrapKey(WrappedKey wrappedKey, Long hsmProfileId) throws KMSExce hsmProfileId = resolveProfileId(wrappedKey.getKekId()); } - return executeWithSession(hsmProfileId, - session -> session.unwrapKey(wrappedKey.getWrappedKeyMaterial(), wrappedKey.getKekId())); + // The blob is unwrapped with the algorithm it was wrapped with, which can differ from the profile's + // current algorithm (e.g. blobs written before the profile was switched, or by an older release). + return executeWithSession(hsmProfileId, session -> session.unwrapKey(wrappedKey.getWrappedKeyMaterial(), + wrappedKey.getKekId(), wrappedKey.getAlgorithm())); } @Override public WrappedKey generateAndWrapDek(KeyPurpose purpose, String kekLabel, int keyBits, Long hsmProfileId) throws KMSException { - byte[] dekBytes = new byte[keyBits / 8]; - new SecureRandom().nextBytes(dekBytes); + if (hsmProfileId == null) { + hsmProfileId = resolveProfileId(kekLabel); + } + byte[] dekBytes = executeWithSession(hsmProfileId, session -> session.generateRandom(keyBits / 8)); try { return wrapKey(dekBytes, purpose, kekLabel, hsmProfileId); @@ -350,6 +388,11 @@ private T executeWithSession(Long hsmProfileId, SessionOperation operatio // isValid() check that discards stale idle sessions on the next poll. if (session != null && e.getErrorType() == KMSException.ErrorType.CONNECTION_FAILED) { invalidateProfileCache(hsmProfileId); + } else if (session != null) { + // Don't reuse a session whose operation failed. PKCS#11 ends an operation when it fails, but some + // HSMs leave it active , and every later operation on that session then fails with CKR_OPERATION_ACTIVE. + pool.discardSession(session); + session = null; } throw e; } finally { @@ -363,7 +406,7 @@ HSMSessionPool getSessionPool(Long profileId) { return sessionPools.computeIfAbsent(profileId, id -> { Map config = loadProfileConfig(id); int maxSessions = Integer.parseInt(config.getOrDefault("max_sessions", "10")); - return new HSMSessionPool(id, maxSessions, this); + return new HSMSessionPool(id, maxSessions, config.get(WRAP_ALGORITHM_DETAIL), this); }); } @@ -406,14 +449,17 @@ private static class HSMSessionPool { private final Long profileId; private final PKCS11HSMProvider provider; private final int maxSessions; + // Algorithm used to wrap new keys under this profile. Profile details cannot be changed after creation. + private final String wrapAlgorithm; // Counts total sessions (idle + active). Acquired on creation, released on close. private final Semaphore sessionPermits; private volatile boolean invalidated = false; - HSMSessionPool(Long profileId, int maxSessions, PKCS11HSMProvider provider) { + HSMSessionPool(Long profileId, int maxSessions, String wrapAlgorithm, PKCS11HSMProvider provider) { this.profileId = profileId; this.provider = provider; this.maxSessions = maxSessions; + this.wrapAlgorithm = wrapAlgorithm; this.sessionPermits = new Semaphore(maxSessions); this.availableSessions = new ArrayBlockingQueue<>(maxSessions); } @@ -476,6 +522,14 @@ void releaseSession(PKCS11Session session) { sessionPermits.release(); } + /** + * Closes a checked-out session instead of returning it to the pool, so the next caller gets a new one. + */ + void discardSession(PKCS11Session session) { + session.close(); + sessionPermits.release(); + } + /** * Marks the pool as invalidated and closes all idle sessions. * Any session currently checked out will be closed (and its permit released) when @@ -498,8 +552,8 @@ void invalidate() { *

Key operations supported: *

    *
  • Key generation: Generate AES keys directly in the HSM
    • - *
  • Key wrapping: Encrypt DEKs using KEKs stored in the HSM (AES-CBC/PKCS5Padding)
    • - *
  • Key unwrapping: Decrypt DEKs using KEKs stored in the HSM (AES-CBC/PKCS5Padding)
    • + *
  • Key wrapping: Encrypt DEKs using KEKs stored in the HSM (AES-GCM, AES-KWP or legacy AES-CBC)
    • + *
  • Key unwrapping: Decrypt DEKs using KEKs stored in the HSM (AES-GCM, AES-KWP or legacy AES-CBC)
    • *
  • Key deletion: Remove keys from the HSM
    • *
  • Key existence check: Verify if a key exists in the HSM
    • *
@@ -507,7 +561,7 @@ void invalidate() { *

Configuration requirements: *

    *
  • {@code library}: Path to PKCS#11 library (required)
    • - *
  • {@code slot} or {@code token_label}: HSM slot/token selection (at least one required)
    • + *
  • {@code slot} or {@code slot_list_index}: HSM slot selection (at least one required)
    • *
  • {@code pin}: PIN for HSM authentication (required, sensitive)
    • *
* @@ -515,7 +569,7 @@ void invalidate() { * {@link KMSException.ErrorType} values for proper retry logic and error reporting. */ private static class PKCS11Session { - private static final int IV_LENGTH = 16; // 128 bits for CBC mode + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); // Counter to bypass JDK SunPKCS11 caching when the HSM restarts. The JDK's // PKCS11.getInstance caches the native module in a static map keyed by the library @@ -534,7 +588,7 @@ private static class PKCS11Session { * The config map (including any sensitive values such as the PIN) is used only * during connection setup and is not retained as a field. * - * @param config HSM profile configuration containing library, slot/token_label, and pin + * @param config HSM profile configuration containing library, slot/slot_list_index, and pin * @throws KMSException if connection fails or configuration is invalid */ PKCS11Session(Map config) throws KMSException { @@ -546,7 +600,7 @@ private static class PKCS11Session { * *

This method: *

    - *
  1. Validates required configuration (library, slot/token_label, pin)
    2. + *
  2. Validates required configuration (library, slot/slot_list_index, pin)
    3. *
  3. Creates a SunPKCS11 provider with the HSM library
    4. *
  4. Loads the PKCS#11 KeyStore
    5. *
  5. Authenticates using the provided PIN
    6. @@ -554,7 +608,7 @@ private static class PKCS11Session { * *

    Slot/token selection: *

      - *
    • If {@code token_label} is provided, it is used (more reliable)
      • + *
    • If {@code slot_list_index} is provided, it is used
      • *
    • Otherwise, {@code slot} (numeric ID) is used
      • *
    * @@ -621,6 +675,9 @@ private void connect(Map config) throws KMSException { connected = true; logger.debug("Successfully connected to PKCS#11 HSM at {}", config.get("library")); + } catch (KMSException e) { + // Already specific, e.g. a missing PIN or slot + throw e; } catch (KeyStoreException | CertificateException | NoSuchAlgorithmException e) { handlePKCS11Exception(e, "Failed to initialize PKCS#11 connection"); } catch (IOException e) { @@ -675,18 +732,15 @@ private String buildSunPKCS11Config(Map config, String nameSuffi configBuilder.append("name=CloudStackHSM-").append(nameSuffix).append("\n"); configBuilder.append("library=").append(libraryPath).append("\n"); - String tokenLabel = config.get("token_label"); String slotListIndex = config.get("slot_list_index"); String slot = config.get("slot"); - if (StringUtils.isNotBlank(tokenLabel)) { - configBuilder.append("tokenLabel=").append(tokenLabel).append("\n"); - } else if (StringUtils.isNotBlank(slotListIndex)) { + if (StringUtils.isNotBlank(slotListIndex)) { configBuilder.append("slotListIndex=").append(slotListIndex).append("\n"); } else if (StringUtils.isNotBlank(slot)) { configBuilder.append("slot=").append(slot).append("\n"); } else { - throw KMSException.invalidParameter("One of 'slot', 'slot_list_index', or 'token_label' is required"); + throw KMSException.invalidParameter("One of 'slot' or 'slot_list_index' is required"); } // Explicitly configure SunPKCS11 to generate AES keys as Data Encryption Keys. @@ -925,60 +979,112 @@ private void validateKeySize(int keyBits) throws KMSException { } } + /** + * Generates random bytes with the token's random number generator (C_GenerateRandom), so that keys come from + * the HSM. Falls back to the JVM's {@link SecureRandom} if the token has no random number generator. + * + * @param length Number of bytes to generate + * @return Random bytes; the caller zeroizes them once they're no longer needed + */ + byte[] generateRandom(int length) { + SecureRandom random; + try { + random = SecureRandom.getInstance("PKCS11", provider); + } catch (NoSuchAlgorithmException e) { + // SunPKCS11 only offers SecureRandom when the token reports a random number generator + logger.debug("HSM token has no random number generator, using the JVM's SecureRandom"); + random = SECURE_RANDOM; + } + byte[] bytes = new byte[length]; + random.nextBytes(bytes); + return bytes; + } + /** * Wraps (encrypts) a plaintext DEK using a KEK stored in the HSM. * - *

    Uses AES-CBC with PKCS5Padding (FIPS 197 + NIST SP 800-38A): + *

    Blob format by algorithm: *

      - *
    • Generates a random 128-bit IV
      • - *
    • Encrypts the DEK using AES-CBC with the KEK from HSM
      • - *
    • Returns format: [IV (16 bytes)][ciphertext]
      • + *
    • {@code AES/GCM/NoPadding}: [IV (12 bytes)][ciphertext][tag (16 bytes)]
      • + *
    • {@code AES/KWP/NoPadding}: [wrapped key] (no IV)
      • + *
    • {@code AES/CBC/PKCS5Padding}: [IV (16 bytes)][ciphertext] (legacy, unauthenticated)
      • *
    * *

    Security: The plaintext DEK should be zeroized by the caller after wrapping. * - * @param plainDek Plaintext DEK to wrap (will be encrypted) - * @param kekLabel Label of the KEK stored in the HSM - * @return Wrapped key blob: [IV][ciphertext] + * @param plainDek Plaintext DEK to wrap (will be encrypted) + * @param kekLabel Label of the KEK stored in the HSM + * @param algorithm Wrap algorithm, one of {@link #ALG_GCM}, {@link #ALG_KWP} or {@link #ALG_CBC} + * @return Wrapped key blob * @throws KMSException with appropriate ErrorType: *

      - *
    • {@code INVALID_PARAMETER} if plainDek is null or empty
      • + *
    • {@code INVALID_PARAMETER} if plainDek is null or empty, or the algorithm is unsupported
      • *
    • {@code KEK_NOT_FOUND} if KEK with label doesn't exist or is not accessible
      • *
    • {@code WRAP_UNWRAP_FAILED} if wrapping operation fails
      • *
    */ - byte[] wrapKey(byte[] plainDek, String kekLabel) throws KMSException { + byte[] wrapKey(byte[] plainDek, String kekLabel, String algorithm) throws KMSException { if (plainDek == null || plainDek.length == 0) { throw KMSException.invalidParameter("Plain DEK cannot be null or empty"); } + int ivLength = ivLength(algorithm); SecretKey kek = getKekFromKeyStore(kekLabel); try { - byte[] iv = new byte[IV_LENGTH]; - new SecureRandom().nextBytes(iv); + byte[] iv = new byte[ivLength]; + SECURE_RANDOM.nextBytes(iv); - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM, provider); - cipher.init(Cipher.ENCRYPT_MODE, kek, new IvParameterSpec(iv)); + Cipher cipher = Cipher.getInstance(algorithm, provider); + initCipher(cipher, Cipher.ENCRYPT_MODE, kek, algorithm, iv); byte[] ciphertext = cipher.doFinal(plainDek); - byte[] result = new byte[IV_LENGTH + ciphertext.length]; - System.arraycopy(iv, 0, result, 0, IV_LENGTH); - System.arraycopy(ciphertext, 0, result, IV_LENGTH, ciphertext.length); + byte[] result = new byte[ivLength + ciphertext.length]; + System.arraycopy(iv, 0, result, 0, ivLength); + System.arraycopy(ciphertext, 0, result, ivLength, ciphertext.length); - logger.debug("Wrapped key with KEK '{}' using AES-CBC", kekLabel); + logger.debug("Wrapped key with KEK '{}' using {}", kekLabel, algorithm); return result; } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeyException e) { handlePKCS11Exception(e, "Invalid key or data for wrapping"); } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { - handlePKCS11Exception(e, "AES-CBC not supported by HSM"); + handlePKCS11Exception(e, algorithm + " not supported by HSM"); } catch (InvalidAlgorithmParameterException e) { - handlePKCS11Exception(e, "Invalid IV for CBC mode"); + handlePKCS11Exception(e, "Invalid parameters for " + algorithm); } catch (Exception e) { handlePKCS11Exception(e, "Failed to wrap key with HSM"); } return null; } + private static boolean isEncryptedDataError(ProviderException e) { + String message = e.getMessage() + " " + (e.getCause() != null ? e.getCause().getMessage() : ""); + return message.contains("CKR_ENCRYPTED_DATA_INVALID") || message.contains("CKR_ENCRYPTED_DATA_LEN_RANGE"); + } + + private static int ivLength(String algorithm) throws KMSException { + if (ALG_GCM.equals(algorithm)) { + return GCM_IV_LENGTH; + } + if (ALG_CBC.equals(algorithm)) { + return AES_BLOCK_LENGTH; + } + if (ALG_KWP.equals(algorithm)) { + return 0; + } + throw KMSException.invalidParameter("Unsupported wrap algorithm: " + algorithm); + } + + private static void initCipher(Cipher cipher, int mode, SecretKey kek, String algorithm, byte[] iv) + throws InvalidKeyException, InvalidAlgorithmParameterException { + if (ALG_GCM.equals(algorithm)) { + cipher.init(mode, kek, new GCMParameterSpec(GCM_TAG_BITS, iv)); + } else if (ALG_CBC.equals(algorithm)) { + cipher.init(mode, kek, new IvParameterSpec(iv)); + } else { + cipher.init(mode, kek); + } + } + /** * Retrieves a KEK (Key Encryption Key) from the HSM KeyStore. * @@ -1015,57 +1121,65 @@ private SecretKey getKekFromKeyStore(String kekLabel) throws KMSException { * Unwraps (decrypts) a wrapped DEK using a KEK stored in the HSM. * *

    - * Uses AES-CBC with PKCS5Padding. Expected format: [IV (16 bytes)][ciphertext]. + * The blob is decrypted with the algorithm it was wrapped with; see + * {@link #wrapKey(byte[], String, String)} for the format of each algorithm. * *

    * Security: The returned plaintext DEK must be zeroized by the caller after * use. * - * @param wrappedBlob Wrapped DEK blob (IV + ciphertext) + * @param wrappedBlob Wrapped DEK blob * @param kekLabel Label of the KEK stored in the HSM + * @param algorithm Algorithm the blob was wrapped with * @return Plaintext DEK * @throws KMSException with appropriate ErrorType: *

      *
    • {@code INVALID_PARAMETER} if wrappedBlob is null, - * empty, or too short
      • + * empty, or too short, or the algorithm is unsupported *
    • {@code KEK_NOT_FOUND} if KEK with label doesn't * exist or is not accessible
      • *
    • {@code WRAP_UNWRAP_FAILED} if unwrapping fails
      • *
    */ - byte[] unwrapKey(byte[] wrappedBlob, String kekLabel) throws KMSException { + byte[] unwrapKey(byte[] wrappedBlob, String kekLabel, String algorithm) throws KMSException { if (wrappedBlob == null || wrappedBlob.length == 0) { throw KMSException.invalidParameter("Wrapped blob cannot be null or empty"); } + int ivLength = ivLength(algorithm); - // Minimum size: IV (16 bytes) + at least one AES block (16 bytes) - if (wrappedBlob.length < IV_LENGTH + 16) { + // Minimum size: IV + at least one AES block (the GCM tag and the smallest KWP output are one block) + if (wrappedBlob.length < ivLength + AES_BLOCK_LENGTH) { throw KMSException.invalidParameter("Wrapped blob too short: expected at least " + - (IV_LENGTH + 16) + " bytes"); + (ivLength + AES_BLOCK_LENGTH) + " bytes"); } SecretKey kek = getKekFromKeyStore(kekLabel); try { - byte[] iv = new byte[IV_LENGTH]; - System.arraycopy(wrappedBlob, 0, iv, 0, IV_LENGTH); - byte[] ciphertext = new byte[wrappedBlob.length - IV_LENGTH]; - System.arraycopy(wrappedBlob, IV_LENGTH, ciphertext, 0, ciphertext.length); + byte[] iv = Arrays.copyOfRange(wrappedBlob, 0, ivLength); + byte[] ciphertext = Arrays.copyOfRange(wrappedBlob, ivLength, wrappedBlob.length); - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM, provider); - cipher.init(Cipher.DECRYPT_MODE, kek, new IvParameterSpec(iv)); + Cipher cipher = Cipher.getInstance(algorithm, provider); + initCipher(cipher, Cipher.DECRYPT_MODE, kek, algorithm, iv); byte[] plainDek = cipher.doFinal(ciphertext); - logger.debug("Unwrapped key with KEK '{}' using AES-CBC", kekLabel); + logger.debug("Unwrapped key with KEK '{}' using {}", kekLabel, algorithm); return plainDek; - } catch (BadPaddingException e) { - throw KMSException.wrapUnwrapFailed( - "Decryption failed: wrapped key may be corrupted or KEK is incorrect", e); - } catch (IllegalBlockSizeException | InvalidKeyException e) { - handlePKCS11Exception(e, "Invalid key or data for unwrapping"); + } catch (BadPaddingException | IllegalBlockSizeException e) { + // Report every padding, tag or block failure the same way and without the cause, so the outcome + // of an unwrap of a modified blob can't be used as a padding oracle (API error, event or log). + throw KMSException.wrapUnwrapFailed("Failed to unwrap key"); + } catch (ProviderException e) { + // An HSM that removes the padding itself (CKM_AES_CBC_PAD) reports bad padding this way + if (isEncryptedDataError(e)) { + throw KMSException.wrapUnwrapFailed("Failed to unwrap key"); + } + handlePKCS11Exception(e, "Failed to unwrap key with HSM"); + } catch (InvalidKeyException e) { + handlePKCS11Exception(e, "Invalid key for unwrapping"); } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { - handlePKCS11Exception(e, "AES-CBC not supported by HSM"); + handlePKCS11Exception(e, algorithm + " not supported by HSM"); } catch (InvalidAlgorithmParameterException e) { - handlePKCS11Exception(e, "Invalid IV for CBC mode"); + handlePKCS11Exception(e, "Invalid parameters for " + algorithm); } catch (Exception e) { handlePKCS11Exception(e, "Failed to unwrap key with HSM"); } @@ -1104,6 +1218,9 @@ void deleteKey(String label) throws KMSException { } else { handlePKCS11Exception(e, "Failed to delete key from HSM"); } + } catch (KMSException e) { + // e.g. KEK_NOT_FOUND from above, which callers rely on + throw e; } catch (Exception e) { handlePKCS11Exception(e, "Failed to delete key from HSM"); } diff --git a/plugins/kms/pkcs11/src/test/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProviderTest.java b/plugins/kms/pkcs11/src/test/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProviderTest.java index 7d539b41fb2c..d51857c46c1b 100644 --- a/plugins/kms/pkcs11/src/test/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProviderTest.java +++ b/plugins/kms/pkcs11/src/test/java/org/apache/cloudstack/kms/provider/pkcs11/PKCS11HSMProviderTest.java @@ -32,12 +32,14 @@ import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -140,8 +142,10 @@ public void testLoadProfileConfig_DecryptsSensitiveValues() { // Verify assertNotNull("Config should not be null", config); - assertEquals(3, config.size()); + assertEquals(4, config.size()); assertEquals("/path/to/lib.so", config.get("library")); + // Not configured, so the default + assertEquals(PKCS11HSMProvider.ALG_GCM, config.get(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL)); // Note: In real code, DBEncryptionUtil.decrypt would be called // Here we just verify the structure is correct assertTrue("Config should contain pin", config.containsKey("pin")); @@ -162,6 +166,91 @@ public void testLoadProfileConfig_HandlesEmptyDetails() { Map config = provider.loadProfileConfig(testProfileId); } + @Test + public void testValidateProfileConfig_DefaultsWrapAlgorithmToGcm() { + Map config = validConfig(); + + provider.validateProfileConfig(config); + + assertEquals(PKCS11HSMProvider.ALG_GCM, config.get(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL)); + } + + @Test + public void testValidateProfileConfig_AcceptsLegacyCbc() { + Map config = validConfig(); + config.put(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL, PKCS11HSMProvider.ALG_CBC); + + provider.validateProfileConfig(config); + + assertEquals(PKCS11HSMProvider.ALG_CBC, config.get(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL)); + } + + @Test(expected = KMSException.class) + public void testValidateProfileConfig_RejectsUnknownWrapAlgorithm() { + Map config = validConfig(); + config.put(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL, "AES/ECB/PKCS5Padding"); + + provider.validateProfileConfig(config); + } + + @Test + public void testValidateProfileConfig_KwpNeedsJava18() { + Map config = validConfig(); + config.put(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL, PKCS11HSMProvider.ALG_KWP); + + if (Runtime.version().feature() >= 18) { + provider.validateProfileConfig(config); + assertEquals(PKCS11HSMProvider.ALG_KWP, config.get(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL)); + } else { + assertThrows(KMSException.class, () -> provider.validateProfileConfig(config)); + } + } + + @Test(expected = KMSException.class) + public void testValidateProfileConfig_RequiresSlotSelection() { + Map config = validConfig(); + config.remove("slot"); + + provider.validateProfileConfig(config); + } + + @Test + public void testValidateProfileConfig_AcceptsSlotListIndex() { + Map config = validConfig(); + config.remove("slot"); + config.put("slot_list_index", "0"); + + provider.validateProfileConfig(config); + } + + @Test(expected = KMSException.class) + public void testGetWrapAlgorithm_RequiresProfileId() { + provider.getWrapAlgorithm(null); + } + + @Test + public void testGetWrapAlgorithm_UsesProfileDetail() { + HSMProfileDetailsVO library = mock(HSMProfileDetailsVO.class); + when(library.getName()).thenReturn("library"); + when(library.getValue()).thenReturn("/path/to/lib.so"); + HSMProfileDetailsVO slot = mock(HSMProfileDetailsVO.class); + when(slot.getName()).thenReturn("slot"); + when(slot.getValue()).thenReturn("0"); + HSMProfileDetailsVO wrapAlgorithm = mock(HSMProfileDetailsVO.class); + when(wrapAlgorithm.getName()).thenReturn(PKCS11HSMProvider.WRAP_ALGORITHM_DETAIL); + when(wrapAlgorithm.getValue()).thenReturn(PKCS11HSMProvider.ALG_CBC); + when(hsmProfileDetailsDao.listByProfileId(testProfileId)).thenReturn(Arrays.asList(library, slot, wrapAlgorithm)); + + assertEquals(PKCS11HSMProvider.ALG_CBC, provider.getWrapAlgorithm(testProfileId)); + } + + private Map validConfig() { + Map config = new HashMap<>(); + config.put("library", "/path/to/lib.so"); + config.put("slot", "0"); + return config; + } + /** * Test: isSensitiveKey correctly identifies sensitive keys */ diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 38ddd52d4a08..bf372a9bc5eb 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2276,6 +2276,12 @@ public void cleanupStorage(boolean recurring) { } } cleanupInactiveTemplates(); + + try { + volService.releaseKeysOfRemovedVolumes(); + } catch (Exception e) { + logger.error("Failed to release the encryption keys of removed volumes", e); + } } finally { scanLock.unlock(); } diff --git a/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java b/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java index cfb16c647fec..27041b87814d 100644 --- a/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/kms/KMSManagerImpl.java @@ -33,8 +33,10 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.projects.Project.ListProjectResourcesCriteria; +import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -85,11 +87,15 @@ import org.apache.logging.log4j.Logger; import javax.inject.Inject; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -105,6 +111,12 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable private static final Logger logger = LogManager.getLogger(KMSManagerImpl.class); private static final Map kmsProviderMap = new HashMap<>(); private static final String DATABASE_PROVIDER_NAME = "database"; + // Wrap algorithms that detect a wrong KEK or a modified blob. + private static final Set AUTHENTICATED_WRAP_ALGORITHMS = Set.of("AES/GCM/NoPadding", "AES/KWP/NoPadding"); + private static final Set VALID_DEK_SIZE_BITS = Set.of(128, 192, 256); + // 4.23 migrated the base64 text of the legacy passphrase (64 characters) instead of its 48 raw bytes. + private static final int LEGACY_MIGRATED_KEY_LENGTH = 64; + private static final int LEGACY_PASSPHRASE_LENGTH = 48; private ExecutorService kmsOperationExecutor; @Inject @@ -137,8 +149,13 @@ public class KMSManagerImpl extends ManagerBase implements KMSManager, Pluggable @Inject private PassphraseDao passphraseDao; + @Inject + private SnapshotDao snapshotDao; private List kmsProviders; private ScheduledExecutorService rewrapExecutor; + private ScheduledExecutorService keyFormatRepairExecutor; + // Id of the last key checked by the key format repair; only used by its single thread + private long keyFormatRepairCursor = 0; @Override @@ -218,24 +235,34 @@ public byte[] unwrapKey(Long wrappedKeyId) throws KMSException { Exception lastException = null; - if (wrappedVO.getKekVersionId() != null) { - KMSKekVersionVO version = kmsKekVersionDao.findById(wrappedVO.getKekVersionId()); - if (version != null && version.getStatus() != KMSKekVersionVO.Status.Archived) { - HSMProfileVO hsmProfile = hsmProfileDao.findById(version.getHsmProfileId()); - if (!hsmProfile.isEnabled()) { - throw KMSException.providerNotInitialized("HSM profile is not enabled for wrapped key: " + version); - } - try { - byte[] dek = getUnwrappedKey(wrappedVO, kmsKey, version, hsmProfile); - - CallContext.current().setEventResourceId(kmsKey.getId()); - CallContext.current().setEventResourceType(ApiCommandResourceType.KmsKey); - CallContext.current().setEventDetails(String.format( - "Unwrapped %s key (wrapped key uuid: %s) using KMS key: %s (uuid: %s) with KEK version %d", - kmsKey.getPurpose().getName(), wrappedVO.getUuid(), kmsKey.getName(), kmsKey.getUuid(), version.getVersionNumber())); + KMSKekVersionVO recordedVersion = wrappedVO.getKekVersionId() != null + ? kmsKekVersionDao.findById(wrappedVO.getKekVersionId()) : null; + if (recordedVersion != null && recordedVersion.getStatus() == KMSKekVersionVO.Status.Archived) { + // The rewrap worker deleted its KEK; no other KEK can unwrap the key + throw KMSException.kekNotFound(String.format("KEK version %d of wrapped key %s is archived", + recordedVersion.getVersionNumber(), wrappedVO.getUuid())); + } + if (recordedVersion != null) { + HSMProfileVO hsmProfile = getEnabledHsmProfile(recordedVersion); + try { + return unwrapKeyWithVersion(wrappedVO, kmsKey, recordedVersion, hsmProfile, ""); + } catch (Exception e) { + lastException = e; + logger.warn("Failed to unwrap with version {}: {}", recordedVersion.getVersionNumber(), e.getMessage(), e); + } + } - logger.debug("Successfully unwrapped key {} with KEK version {}", wrappedKeyId, version.getVersionNumber()); - return dek; + // Other versions only for an authenticated algorithm: with CBC another KEK can unwrap the blob to a wrong + // key without an error, and a rewrap would then lose the real key for good. + if (AUTHENTICATED_WRAP_ALGORITHMS.contains(wrappedVO.getWrapAlgorithm())) { + List otherVersions = kmsKekVersionDao.getVersionsForDecryption(kmsKey.getId()).stream() + .filter(version -> recordedVersion == null || !version.getId().equals(recordedVersion.getId())) + .sorted(Comparator.comparing(KMSKekVersionVO::getVersionNumber).reversed()) + .collect(Collectors.toList()); + for (KMSKekVersionVO version : otherVersions) { + HSMProfileVO hsmProfile = getEnabledHsmProfile(version); + try { + return unwrapKeyWithVersion(wrappedVO, kmsKey, version, hsmProfile, " (fallback)"); } catch (Exception e) { lastException = e; logger.warn("Failed to unwrap with version {}: {}", version.getVersionNumber(), e.getMessage(), e); @@ -243,32 +270,34 @@ public byte[] unwrapKey(Long wrappedKeyId) throws KMSException { } } - // Fallback: try all available versions for decryption - List versions = kmsKekVersionDao.getVersionsForDecryption(kmsKey.getId()); - for (KMSKekVersionVO version : versions) { - HSMProfileVO hsmProfile = hsmProfileDao.findById(version.getHsmProfileId()); - if (!hsmProfile.isEnabled()) { - throw KMSException.providerNotInitialized("HSM profile is not enabled for wrapped key: " + version); - } - try { - byte[] dek = getUnwrappedKey(wrappedVO, kmsKey, version, hsmProfile); - - CallContext.current().setEventResourceId(kmsKey.getId()); - CallContext.current().setEventResourceType(ApiCommandResourceType.KmsKey); - CallContext.current().setEventDetails(String.format( - "Unwrapped %s key (wrapped key uuid: %s) using KMS key: %s (uuid: %s) with KEK version %d (fallback)", - kmsKey.getPurpose().getName(), wrappedVO.getUuid(), kmsKey.getName(), kmsKey.getUuid(), version.getVersionNumber())); + if (lastException == null) { + // Nothing to try, e.g. the KEK version was archived and its KEK deleted + throw KMSException.kekNotFound("No usable KEK version for wrapped key " + wrappedKeyId); + } + throw KMSException.wrapUnwrapFailed("Failed to unwrap key: no available KEK version for wrapped key " + wrappedKeyId, lastException); + } - logger.info("Successfully unwrapped key {} with KEK version {} (fallback)", - wrappedKeyId, version.getVersionNumber()); - return dek; - } catch (Exception e) { - lastException = e; - logger.warn("Failed to unwrap with version {}: {}", version.getVersionNumber(), e.getMessage(), e); - } + private HSMProfileVO getEnabledHsmProfile(KMSKekVersionVO version) { + HSMProfileVO hsmProfile = hsmProfileDao.findById(version.getHsmProfileId()); + if (!hsmProfile.isEnabled()) { + throw KMSException.providerNotInitialized("HSM profile is not enabled for wrapped key: " + version); } + return hsmProfile; + } - throw KMSException.wrapUnwrapFailed("Failed to unwrap key: no available KEK version for wrapped key " + wrappedKeyId, lastException); + private byte[] unwrapKeyWithVersion(KMSWrappedKeyVO wrappedVO, KMSKeyVO kmsKey, KMSKekVersionVO version, + HSMProfileVO hsmProfile, String fallback) throws Exception { + byte[] dek = getUnwrappedKey(wrappedVO, kmsKey, version, hsmProfile); + + CallContext.current().setEventResourceId(kmsKey.getId()); + CallContext.current().setEventResourceType(ApiCommandResourceType.KmsKey); + CallContext.current().setEventDetails(String.format( + "Unwrapped %s key (wrapped key uuid: %s) using KMS key: %s (uuid: %s) with KEK version %d%s", + kmsKey.getPurpose().getName(), wrappedVO.getUuid(), kmsKey.getName(), kmsKey.getUuid(), + version.getVersionNumber(), fallback)); + + logger.debug("Successfully unwrapped key {} with KEK version {}{}", wrappedVO.getId(), version.getVersionNumber(), fallback); + return dek; } private byte[] getUnwrappedKey(KMSWrappedKeyVO wrappedVO, KMSKeyVO kmsKey, @@ -276,7 +305,7 @@ private byte[] getUnwrappedKey(KMSWrappedKeyVO wrappedVO, KMSKeyVO kmsKey, KMSProvider provider = getKMSProvider(hsmProfile.getProtocol()); WrappedKey wrapped = new WrappedKey(wrappedVO.getUuid(), version.getKekLabel(), kmsKey.getPurpose(), - kmsKey.getAlgorithm(), wrappedVO.getWrappedBlob(), + wrappedVO.getWrapAlgorithm(), wrappedVO.getWrappedBlob(), hsmProfile.getProtocol(), wrappedVO.getCreated(), kmsKey.getZoneId()); return retryOperation(() -> provider.unwrapKey(wrapped, version.getHsmProfileId())); } @@ -308,6 +337,9 @@ public WrappedKey generateVolumeKeyWithKek(KMSKey kmsKey, Long callerAccountId) KMSProvider provider = getKMSProvider(hsmProfile.getProtocol()); int dekSize = KMSDekSizeBits.value(); + if (!VALID_DEK_SIZE_BITS.contains(dekSize)) { + throw KMSException.invalidParameter(KMSDekSizeBits.key() + " must be 128, 192 or 256: " + dekSize); + } WrappedKey wrappedKey; try { wrappedKey = retryOperation(() -> provider.generateAndWrapDek(KeyPurpose.VOLUME_ENCRYPTION, @@ -315,6 +347,7 @@ public WrappedKey generateVolumeKeyWithKek(KMSKey kmsKey, Long callerAccountId) activeVersion.getHsmProfileId())); KMSWrappedKeyVO wrappedKeyVO = new KMSWrappedKeyVO(kmsKey.getId(), activeVersion.getId(), kmsKey.getZoneId(), wrappedKey.getWrappedKeyMaterial()); + wrappedKeyVO.setWrapAlgorithm(wrappedKey.getAlgorithm()); wrappedKeyVO = kmsWrappedKeyDao.persist(wrappedKeyVO); // Volume creation code looks up by UUID and sets volume.kmsWrappedKeyId @@ -442,10 +475,10 @@ KMSKey createUserKMSKey(Long accountId, Long domainId, Long zoneId, throw new InvalidParameterValueException("A KMS key with name " + name + " already exists in this account"); } + KMSProvider provider = getKMSProvider(profile.getProtocol()); KMSKeyVO kmsKey = new KMSKeyVO(name, description, "", purpose, - accountId, domainId, zoneId, "AES/GCM/NoPadding", keyBits); + accountId, domainId, zoneId, provider.getWrapAlgorithm(hsmProfileId), keyBits); - KMSProvider provider = getKMSProvider(profile.getProtocol()); String kekLabel = purpose.generateKekLabel(domainId, accountId, kmsKey.getUuid(), 1); String providerKekLabel; @@ -455,6 +488,7 @@ KMSKey createUserKMSKey(Long accountId, Long domainId, Long zoneId, } catch (Exception e) { throw handleKmsException(e); } + verifyKekCanWrap(provider, purpose, providerKekLabel, profile); kmsKey.setKekLabel(providerKekLabel); kmsKey.setHsmProfileId(finalProfileId); @@ -591,18 +625,35 @@ public boolean deleteKMSWrappedKey(Volume vol) throws KMSException { if (CollectionUtils.isNotEmpty(volumes)) { for (VolumeVO volume : volumes) { if (volume.getId() != vol.getId()) { - logger.warn("KMS Wrapped Key {} is still in use by volume {}, cannot delete", + // Normal after a volume migration, the source and the new volume share the key + logger.debug("KMS Wrapped Key {} is still in use by volume {}, cannot delete", wrappedKey, volume); return false; } } } + if (isNeededBySnapshots(wrappedKey)) { + logger.debug("KMS Wrapped Key {} is still needed by snapshots of volume {} or of another volume " + + "sharing it, keeping it", wrappedKey, vol); + return false; + } logger.info("Deleted KMS Wrapped Key {} for volume {}", wrappedKey, vol); return kmsWrappedKeyDao.remove(wrappedKey.getId()); } return true; } + /** + * Whether a snapshot that isn't destroyed exists for any volume (removed volumes included) that uses the + * wrapped key. Restoring or deleting such a snapshot needs the key of its volume. + */ + private boolean isNeededBySnapshots(KMSWrappedKeyVO wrappedKey) { + List volumeIds = volumeDao.listIncludingRemovedByKmsWrappedKeyId(wrappedKey.getId()).stream() + .map(VolumeVO::getId).collect(Collectors.toList()); + return snapshotDao.searchByVolumes(volumeIds).stream() + .anyMatch(snapshot -> snapshot.getState() != Snapshot.State.Destroyed); + } + @Override public SuccessResponse deleteKMSKey(DeleteKMSKeyCmd cmd) throws KMSException { Account caller = CallContext.current().getCallingAccount(); @@ -638,12 +689,21 @@ void deleteUserKMSKey(KMSKeyVO key) throws KMSException { kekVersion.getKekLabel(), kekVersion.getVersionNumber(), provider.getProviderName()); } } catch (Exception e) { + if (e instanceof KMSException && ((KMSException) e).getErrorType() == KMSException.ErrorType.KEK_NOT_FOUND) { + // e.g. an archived version whose KEK the rewrap worker already deleted + logger.info("KEK {} (v{}) is already gone from the provider", kekVersion.getKekLabel(), + kekVersion.getVersionNumber()); + continue; + } logger.warn("Failed to delete KEK {} (v{}) from provider during KMS key deletion: {}", kekVersion.getKekLabel(), kekVersion.getVersionNumber(), e.getMessage()); ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), key.getAccountId(), EventVO.LEVEL_WARN, EventTypes.EVENT_KMS_KEY_DELETE, true, String.format("Failed to delete KEK %s from provider during KMS key deletion", kekVersion.getKekLabel()), key.getId(), ApiCommandResourceType.KmsKey.toString(), CallContext.current().getStartEventId()); + // Keep the KMS key so the KEK material isn't left in the HSM without any reference to it + throw new CloudRuntimeException(String.format("Failed to delete KEK %s from provider, KMS key %s was " + + "not deleted: %s", kekVersion.getKekLabel(), key.getUuid(), e.getMessage()), e); } } @@ -726,6 +786,8 @@ String rotateKek(KMSKeyVO kmsKey, String oldKekLabel, String newKekLabel, int ke final HSMProfileVO finalHSMProfile = newHSMProfile; String newKekId = retryOperation( () -> provider.createKek(kmsKey.getPurpose(), finalNewKekLabel, keyBits, newProfileId)); + verifyKekCanWrap(provider, kmsKey.getPurpose(), newKekId, finalHSMProfile); + String newWrapAlgorithm = provider.getWrapAlgorithm(newProfileId); try { KMSKekVersionVO newVersion = Transaction @@ -740,6 +802,7 @@ String rotateKek(KMSKeyVO kmsKey, String oldKekLabel, String newKekLabel, int ke logger.info("Updated KMS key {} to use HSM profile {}", kmsKey, finalHSMProfile); } kmsKey.setKekLabel(finalNewKekLabel); + kmsKey.setAlgorithm(newWrapAlgorithm); kmsKeyDao.update(kmsKey.getId(), kmsKey); return version; }); @@ -755,7 +818,8 @@ String rotateKek(KMSKeyVO kmsKey, String oldKekLabel, String newKekLabel, int ke + "{} from provider {}", kmsKey, newKekId, provider.getProviderName()); try { - provider.deleteKek(newKekId); + // The version row was rolled back, so the provider can't find the KEK's profile by itself. + provider.deleteKek(newKekId, newProfileId); } catch (KMSException ex) { logger.error("Failed to delete orphaned KEK {} from provider {} after DB failure: {}", newKekId, provider.getProviderName(), ex.getMessage()); @@ -773,6 +837,40 @@ String rotateKek(KMSKeyVO kmsKey, String oldKekLabel, String newKekLabel, int ke } } + /** + * Wraps and unwraps a random key with a newly created KEK, so an HSM that doesn't support the profile's wrap + * algorithm (e.g. GCM with a caller-supplied IV in FIPS mode) fails here rather than at the first volume + * creation or rewrap. On failure the KEK is deleted again. + */ + private void verifyKekCanWrap(KMSProvider provider, KeyPurpose purpose, String kekLabel, HSMProfileVO profile) { + String algorithm = null; + byte[] testKey = new byte[32]; + new SecureRandom().nextBytes(testKey); + byte[] unwrapped = null; + try { + algorithm = provider.getWrapAlgorithm(profile.getId()); + WrappedKey wrapped = provider.wrapKey(testKey, purpose, kekLabel, profile.getId()); + unwrapped = provider.unwrapKey(wrapped, profile.getId()); + if (!Arrays.equals(testKey, unwrapped)) { + throw KMSException.wrapUnwrapFailed("unwrapped key does not match the original"); + } + } catch (Exception e) { + try { + provider.deleteKek(kekLabel, profile.getId()); + } catch (Exception ex) { + logger.warn("Failed to delete KEK {} from HSM profile {} after a failed wrap test: {}", kekLabel, + profile, ex.getMessage()); + } + throw KMSException.wrapUnwrapFailed(String.format("HSM profile %s could not wrap and unwrap a key with %s: %s", + profile.getName(), algorithm, e.getMessage()), e); + } finally { + Arrays.fill(testKey, (byte) 0); + if (unwrapped != null) { + Arrays.fill(unwrapped, (byte) 0); + } + } + } + private KMSKekVersionVO createKekVersion(KMSKekVersionVO newVersion) throws KMSException { List existingVersions = kmsKekVersionDao.listByKmsKeyId(newVersion.getKmsKeyId()); int nextVersion = existingVersions.stream() @@ -882,7 +980,7 @@ public int migrateVolumesToKMS(MigrateVolumesToKMSCmd cmd) throws KMSException { return successCount; } - private boolean migrateVolumeToKmsKey(KMSProvider provider, VolumeVO volume, KMSKey kmsKey, + boolean migrateVolumeToKmsKey(KMSProvider provider, VolumeVO volume, KMSKey kmsKey, KMSKekVersionVO activeVersion) { if (volume.getAccountId() != kmsKey.getAccountId()) { throw new InvalidParameterValueException( @@ -897,9 +995,10 @@ private boolean migrateVolumeToKmsKey(KMSProvider provider, VolumeVO volume, KMS return false; } - // PassphraseVO.getPassphrase() returns Base64-encoded bytes matching KVM/QEMU - // format - byte[] passphraseBytes = passphrase.getPassphrase(); + // The legacy passphrase is stored as base64 text, and that text is what qemu/libvirt get as the LUKS + // passphrase. VolumeObject.getPassphrase() base64-encodes an unwrapped KMS key, so wrap the decoded bytes + // for the volume to keep exactly the same passphrase. + byte[] passphraseBytes = Base64.getDecoder().decode(passphrase.getPassphraseString()); try { WrappedKey wrappedKey = provider.wrapKey( passphraseBytes, @@ -907,17 +1006,27 @@ private boolean migrateVolumeToKmsKey(KMSProvider provider, VolumeVO volume, KMS activeVersion.getKekLabel(), activeVersion.getHsmProfileId()); - KMSWrappedKeyVO wrappedKeyVO = new KMSWrappedKeyVO( - kmsKey.getId(), - activeVersion.getId(), - volume.getDataCenterId(), - wrappedKey.getWrappedKeyMaterial()); - wrappedKeyVO = kmsWrappedKeyDao.persist(wrappedKeyVO); - - volume.setKmsWrappedKeyId(wrappedKeyVO.getId()); - volume.setKmsKeyId(kmsKey.getId()); - volume.setPassphraseId(null); - volumeDao.update(volume.getId(), volume); + Long passphraseId = volume.getPassphraseId(); + Transaction.execute((TransactionCallbackWithException) status -> { + KMSWrappedKeyVO wrappedKeyVO = new KMSWrappedKeyVO( + kmsKey.getId(), + activeVersion.getId(), + volume.getDataCenterId(), + wrappedKey.getWrappedKeyMaterial()); + wrappedKeyVO.setWrapAlgorithm(wrappedKey.getAlgorithm()); + wrappedKeyVO = kmsWrappedKeyDao.persist(wrappedKeyVO); + + volume.setKmsWrappedKeyId(wrappedKeyVO.getId()); + volume.setKmsKeyId(kmsKey.getId()); + volume.setPassphraseId(null); + volumeDao.update(volume.getId(), volume); + + // Don't leave the volume key behind in the passphrase table, unless another volume still uses it + if (volumeDao.listIncludingRemovedByPassphraseId(passphraseId).isEmpty()) { + passphraseDao.remove(passphraseId); + } + return null; + }); ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), kmsKey.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VOLUME_MIGRATE_TO_KMS, true, @@ -1059,17 +1168,16 @@ public HSMProfile addHSMProfile(CreateHSMProfileCmd cmd) throws KMSException { profile.setIsPublic(isPublic); profile = hsmProfileDao.persist(profile); - if (cmd.getDetails() != null) { - for (Map.Entry entry : cmd.getDetails().entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); + // The validated details, which include the defaults the provider filled in (e.g. the wrap algorithm) + for (Map.Entry entry : details.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); - if (isSensitiveKey(key)) { - value = DBEncryptionUtil.encrypt(value); - } - - hsmProfileDetailsDao.persist(profile.getId(), key, value); + if (isSensitiveKey(key)) { + value = DBEncryptionUtil.encrypt(value); } + + hsmProfileDetailsDao.persist(profile.getId(), key, value); } CallContext.current().setEventResourceId(profile.getId()); @@ -1331,9 +1439,17 @@ public HSMProfile updateHSMProfile(UpdateHSMProfileCmd cmd) throws KMSException hsmProfileDao.update(profile.getId(), profile); + String testingOnlyWarning = ""; + if (Boolean.TRUE.equals(cmd.getEnabled()) && DATABASE_PROVIDER_NAME.equalsIgnoreCase(profile.getProtocol())) { + testingOnlyWarning = ". The database provider stores KEKs in the CloudStack database and is meant for " + + "testing only, it is not a substitute for an HSM"; + logger.warn("HSM profile {} was enabled{}", profile, testingOnlyWarning); + } + CallContext.current().setEventResourceId(profile.getId()); CallContext.current().setEventResourceType(ApiCommandResourceType.HsmProfile); - CallContext.current().setEventDetails(String.format("Updated HSM profile: %s (uuid: %s)", profile.getName(), profile.getUuid())); + CallContext.current().setEventDetails(String.format("Updated HSM profile: %s (uuid: %s)%s", profile.getName(), + profile.getUuid(), testingOnlyWarning)); return profile; } @@ -1610,6 +1726,7 @@ public boolean start() { } scheduleRewrapWorker(); + scheduleKeyFormatRepair(); return true; } @@ -1768,6 +1885,7 @@ void rewrapSingleKey(KMSWrappedKeyVO wrappedKeyVO, KMSKeyVO kmsKey, wrappedKeyVO.setKekVersionId(newVersion.getId()); wrappedKeyVO.setWrappedBlob(newWrapped.getWrappedKeyMaterial()); + wrappedKeyVO.setWrapAlgorithm(newWrapped.getAlgorithm()); kmsWrappedKeyDao.update(wrappedKeyVO.getId(), wrappedKeyVO); ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), kmsKey.getAccountId(), @@ -1781,12 +1899,174 @@ void rewrapSingleKey(KMSWrappedKeyVO wrappedKeyVO, KMSKeyVO kmsKey, } } + /** + * 4.23 migrated legacy volume passphrases to KMS as their base64 text instead of their raw bytes, so those + * volumes got a wrong passphrase. The upgrade flags every wrapped key that existed at that point; this task + * checks each one once and rewraps the affected ones with the raw bytes. A key that can't be unwrapped (e.g. the + * HSM is down) stays flagged and is retried. The task stops once no flagged key is left. + */ + private void scheduleKeyFormatRepair() { + if (kmsWrappedKeyDao.listFormatCheckPending(0, 1).isEmpty()) { + return; + } + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "KMSKeyFormatRepair"); + t.setDaemon(true); + return t; + }); + keyFormatRepairExecutor = executor; + executor.scheduleWithFixedDelay(new ManagedContextRunnable() { + @Override + protected void runInContext() { + try { + if (repairKeyFormatBatch()) { + logger.info("All KMS wrapped keys flagged at the upgrade from 4.23 have been checked"); + executor.shutdown(); + } + } catch (final Exception e) { + logger.error("Error while checking the format of KMS wrapped keys", e); + } + } + }, 30, 60, TimeUnit.SECONDS); + } + + /** + * Checks the next batch of flagged keys. A batch ends at the first key that can't be checked (e.g. its HSM is + * down), so the lock shared with the rewrap worker isn't held through retries; the next batch resumes after that + * key so it doesn't block the others. + * + * @return true when no flagged key is left + */ + boolean repairKeyFormatBatch() { + GlobalLock lock = GlobalLock.getInternLock("kms.rewrap.worker"); + try { + if (!lock.lock(5)) { + return false; + } + try { + List keys = kmsWrappedKeyDao.listFormatCheckPending(keyFormatRepairCursor, + KMSRewrapBatchSize.value()); + if (keys.isEmpty()) { + boolean done = keyFormatRepairCursor == 0; + keyFormatRepairCursor = 0; + return done; + } + for (KMSWrappedKeyVO key : keys) { + keyFormatRepairCursor = key.getId(); + try { + repairKeyFormat(key); + } catch (Exception e) { + logger.warn("Could not check the format of KMS wrapped key {}, will retry: {}", key, e.getMessage()); + logger.debug("Failed to check the format of KMS wrapped key {}", key, e); + break; + } + } + return false; + } finally { + lock.unlock(); + } + } finally { + lock.releaseRef(); + } + } + + private void repairKeyFormat(KMSWrappedKeyVO wrappedKeyVO) { + byte[] key; + try { + key = unwrapKey(wrappedKeyVO.getId()); + } catch (KMSException e) { + if (e.getErrorType() != KMSException.ErrorType.KEK_NOT_FOUND) { + throw e; + } + // Gone for good, e.g. its KMS key or KEK was deleted, so there is nothing to repair + logger.warn("KMS wrapped key {} can't be unwrapped anymore, not checking its format: {}", wrappedKeyVO, + e.getMessage()); + markFormatChecked(wrappedKeyVO); + return; + } + byte[] passphrase = decodeLegacyMigratedKey(key); + try { + if (passphrase != null) { + rewrapLegacyMigratedKey(wrappedKeyVO, passphrase); + } + markFormatChecked(wrappedKeyVO); + } finally { + Arrays.fill(key, (byte) 0); + if (passphrase != null) { + Arrays.fill(passphrase, (byte) 0); + } + } + } + + private void rewrapLegacyMigratedKey(KMSWrappedKeyVO wrappedKeyVO, byte[] passphrase) { + List volumes = volumeDao.findByKmsWrappedKeyId(wrappedKeyVO.getId()); + List createdAfterMigration = volumes.stream() + .filter(volume -> volume.getCreated().after(wrappedKeyVO.getCreated())) + .collect(Collectors.toList()); + if (!createdAfterMigration.isEmpty()) { + // e.g. the copy made by a storage migration on 4.23, which was encrypted with the passphrase 4.23 derived + // from this key. Repairing the key would make such a volume unreadable, so leave it to the admin. + logger.warn("Not repairing KMS wrapped key {}, which 4.23 migrated in a wrong format: volume(s) {} were " + + "created after the migration and may be encrypted with the passphrase 4.23 used. Check these " + + "volumes manually.", wrappedKeyVO, createdAfterMigration); + return; + } + + KMSKeyVO kmsKey = kmsKeyDao.findById(wrappedKeyVO.getKmsKeyId()); + KMSKekVersionVO activeVersion = getActiveKekVersion(kmsKey.getId()); + HSMProfileVO hsmProfile = hsmProfileDao.findById(activeVersion.getHsmProfileId()); + WrappedKey rewrapped = getKMSProvider(hsmProfile.getProtocol()).wrapKey(passphrase, + kmsKey.getPurpose(), activeVersion.getKekLabel(), activeVersion.getHsmProfileId()); + wrappedKeyVO.setKekVersionId(activeVersion.getId()); + wrappedKeyVO.setWrappedBlob(rewrapped.getWrappedKeyMaterial()); + wrappedKeyVO.setWrapAlgorithm(rewrapped.getAlgorithm()); + + for (VolumeVO volume : volumes) { + logger.info("Repaired the KMS key of volume {}, which was migrated to KMS by 4.23", volume); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), + volume.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VOLUME_MIGRATE_TO_KMS, true, + String.format("Repaired the KMS key of the volume, which was migrated to KMS key %s by 4.23", + kmsKey.getUuid()), + volume.getId(), ApiCommandResourceType.Volume.toString(), CallContext.current().getStartEventId()); + } + } + + private void markFormatChecked(KMSWrappedKeyVO wrappedKeyVO) { + wrappedKeyVO.setFormatCheckPending(false); + kmsWrappedKeyDao.update(wrappedKeyVO.getId(), wrappedKeyVO); + } + + /** + * @return the raw passphrase if {@code key} is the base64 text of a 48-byte legacy passphrase, as 4.23 migrated + * it, otherwise null + */ + static byte[] decodeLegacyMigratedKey(byte[] key) { + if (key.length != LEGACY_MIGRATED_KEY_LENGTH) { + return null; + } + byte[] decoded; + try { + decoded = Base64.getDecoder().decode(key); + } catch (IllegalArgumentException e) { + return null; + } + if (decoded.length != LEGACY_PASSPHRASE_LENGTH || !Arrays.equals(Base64.getEncoder().encode(decoded), key)) { + Arrays.fill(decoded, (byte) 0); + return null; + } + return decoded; + } + @Override public boolean stop() { if (rewrapExecutor != null) { rewrapExecutor.shutdownNow(); rewrapExecutor = null; } + if (keyFormatRepairExecutor != null) { + keyFormatRepairExecutor.shutdownNow(); + keyFormatRepairExecutor = null; + } kmsOperationExecutor.shutdownNow(); return super.stop(); } diff --git a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyCreationTest.java b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyCreationTest.java index 54e9e4d815be..3be5121ff906 100644 --- a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyCreationTest.java +++ b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyCreationTest.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.framework.kms.KMSException; import org.apache.cloudstack.framework.kms.KMSProvider; import org.apache.cloudstack.framework.kms.KeyPurpose; +import org.apache.cloudstack.framework.kms.WrappedKey; import org.apache.cloudstack.kms.dao.HSMProfileDao; import org.apache.cloudstack.kms.dao.KMSKekVersionDao; import org.apache.cloudstack.kms.dao.KMSKeyDao; @@ -41,15 +42,18 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -119,12 +123,14 @@ public void testCreateUserKMSKey_WithExplicitProfile() throws Exception { Long hsmProfileId = 10L; HSMProfileVO profile = mock(HSMProfileVO.class); + when(profile.getId()).thenReturn(hsmProfileId); when(profile.getProtocol()).thenReturn(testProviderName); when(hsmProfileDao.findById(hsmProfileId)).thenReturn(profile); // Mock provider KEK creation when(kmsProvider.createKek(any(KeyPurpose.class), anyString(), anyInt(), eq(hsmProfileId))) .thenReturn("test-kek-label"); + mockWrapRoundTrip(hsmProfileId); // Mock DAO persist operations KMSKeyVO mockKey = mock(KMSKeyVO.class); @@ -154,9 +160,39 @@ public void testCreateUserKMSKey_WithExplicitProfile() throws Exception { verify(kmsKeyDao).persist(keyCaptor.capture()); KMSKeyVO createdKey = keyCaptor.getValue(); assertEquals(hsmProfileId, createdKey.getHsmProfileId()); + // The key shows the algorithm its keys are wrapped with + assertEquals("AES/GCM/NoPadding", createdKey.getAlgorithm()); } } + /** + * Test: createUserKMSKey deletes the new KEK and fails when the HSM can't wrap and unwrap with it + */ + @Test + public void testCreateUserKMSKey_DeletesKekWhenWrapTestFails() { + Long hsmProfileId = 10L; + + HSMProfileVO profile = mock(HSMProfileVO.class); + when(profile.getId()).thenReturn(hsmProfileId); + when(profile.getProtocol()).thenReturn(testProviderName); + when(hsmProfileDao.findById(hsmProfileId)).thenReturn(profile); + + when(kmsProvider.createKek(any(KeyPurpose.class), anyString(), anyInt(), eq(hsmProfileId))) + .thenReturn("test-kek-label"); + when(kmsProvider.getWrapAlgorithm(hsmProfileId)).thenReturn("AES/GCM/NoPadding"); + when(kmsProvider.wrapKey(any(byte[].class), any(KeyPurpose.class), eq("test-kek-label"), eq(hsmProfileId))) + .thenThrow(KMSException.kekOperationFailed("AES/GCM/NoPadding not supported by HSM")); + doReturn(kmsProvider).when(kmsManager).getKMSProvider(testProviderName); + + assertThrows(KMSException.class, () -> kmsManager.createUserKMSKey(testAccountId, testDomainId, + testZoneId, "test-key", "Test key", KeyPurpose.VOLUME_ENCRYPTION, 256, hsmProfileId)); + + // The KEK has no version record yet, so it is deleted with its HSM profile + verify(kmsProvider).deleteKek("test-kek-label", hsmProfileId); + verify(kmsKeyDao, never()).persist(any(KMSKeyVO.class)); + verify(kmsKekVersionDao, never()).persist(any(KMSKekVersionVO.class)); + } + /** * Test: createUserKMSKey throws exception when explicit profile not found */ @@ -180,11 +216,13 @@ public void testCreateUserKMSKey_CreatesKekVersionWithProfileId() throws Excepti Long hsmProfileId = 40L; HSMProfileVO profile = mock(HSMProfileVO.class); + when(profile.getId()).thenReturn(hsmProfileId); when(profile.getProtocol()).thenReturn(testProviderName); when(hsmProfileDao.findById(hsmProfileId)).thenReturn(profile); when(kmsProvider.createKek(any(KeyPurpose.class), anyString(), anyInt(), eq(hsmProfileId))) .thenReturn("test-kek-label"); + mockWrapRoundTrip(hsmProfileId); KMSKeyVO mockKey = mock(KMSKeyVO.class); when(mockKey.getId()).thenReturn(1L); @@ -213,4 +251,18 @@ public void testCreateUserKMSKey_CreatesKekVersionWithProfileId() throws Excepti assertEquals("test-kek-label", createdVersion.getKekLabel()); } } + + /** + * Makes the provider hand back whatever was wrapped, for the wrap test done after a KEK is created. + */ + private void mockWrapRoundTrip(Long hsmProfileId) { + when(kmsProvider.getWrapAlgorithm(hsmProfileId)).thenReturn("AES/GCM/NoPadding"); + AtomicReference wrapped = new AtomicReference<>(); + when(kmsProvider.wrapKey(any(byte[].class), any(KeyPurpose.class), anyString(), eq(hsmProfileId))) + .thenAnswer(invocation -> { + wrapped.set(((byte[]) invocation.getArgument(0)).clone()); + return mock(WrappedKey.class); + }); + when(kmsProvider.unwrapKey(any(WrappedKey.class), eq(hsmProfileId))).thenAnswer(invocation -> wrapped.get().clone()); + } } diff --git a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyLifecycleTest.java b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyLifecycleTest.java index 1d4c2947a88b..b0a7544fea02 100644 --- a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyLifecycleTest.java +++ b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyLifecycleTest.java @@ -19,7 +19,13 @@ import com.cloud.event.ActionEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.framework.kms.KMSException; import org.apache.cloudstack.framework.kms.KMSProvider; import org.apache.cloudstack.framework.kms.KeyPurpose; @@ -32,6 +38,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; @@ -44,8 +51,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -55,6 +64,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -83,6 +93,9 @@ public class KMSManagerImplKeyLifecycleTest { @Mock private VolumeDao volumeDao; + @Mock + private SnapshotDao snapshotDao; + @Mock private KMSProvider kmsProvider; @@ -126,61 +139,175 @@ public void testUnwrapKey_ThrowsWhenKmsKeyNotFound() throws KMSException { @Test public void testUnwrapKey_SucceedsWithHintVersion() { - Long wrappedKeyId = 1L; - Long kmsKeyId = 10L; - Long versionId = 5L; + mockWrappedKey(5L, "AES/CBC/PKCS5Padding"); + mockDatabaseProfile(); + KMSKekVersionVO version = mockVersion(5L, 1, KMSKekVersionVO.Status.Active); + when(kmsKekVersionDao.findById(5L)).thenReturn(version); + when(kmsProvider.unwrapKey(any(WrappedKey.class), anyLong())).thenReturn(new byte[32]); - KMSWrappedKeyVO wrappedVO = mock(KMSWrappedKeyVO.class); - when(wrappedVO.getKmsKeyId()).thenReturn(kmsKeyId); - when(wrappedVO.getKekVersionId()).thenReturn(versionId); - when(wrappedVO.getWrappedBlob()).thenReturn(new byte[]{0, 1}); - when(kmsWrappedKeyDao.findById(wrappedKeyId)).thenReturn(wrappedVO); + byte[] result = kmsManager.unwrapKey(1L); - KMSKeyVO kmsKey = mock(KMSKeyVO.class); - when(kmsKey.getPurpose()).thenReturn(KeyPurpose.VOLUME_ENCRYPTION); - when(kmsKey.getAlgorithm()).thenReturn("AES/GCM/NoPadding"); - when(kmsKeyDao.findById(kmsKeyId)).thenReturn(kmsKey); + assertNotNull(result); + // The blob is unwrapped with the algorithm recorded with it + ArgumentCaptor wrappedKeyCaptor = ArgumentCaptor.forClass(WrappedKey.class); + verify(kmsProvider).unwrapKey(wrappedKeyCaptor.capture(), anyLong()); + assertEquals("AES/CBC/PKCS5Padding", wrappedKeyCaptor.getValue().getAlgorithm()); + verify(kmsKekVersionDao, never()).getVersionsForDecryption(10L); + } - KMSKekVersionVO version = mock(KMSKekVersionVO.class); - when(version.getStatus()).thenReturn(KMSKekVersionVO.Status.Active); - when(version.getHsmProfileId()).thenReturn(20L); - when(version.getKekLabel()).thenReturn("kek-label"); - when(kmsKekVersionDao.findById(versionId)).thenReturn(version); + @Test + public void testUnwrapKey_NoFallbackForCbcWhenRecordedVersionFails() { + mockWrappedKey(5L, "AES/CBC/PKCS5Padding"); + mockDatabaseProfile(); + KMSKekVersionVO version = mockVersion(5L, 1, KMSKekVersionVO.Status.Previous); + when(kmsKekVersionDao.findById(5L)).thenReturn(version); + when(kmsProvider.unwrapKey(any(WrappedKey.class), anyLong())).thenThrow(KMSException.wrapUnwrapFailed("Failed to unwrap key")); + + assertThrows(KMSException.class, () -> kmsManager.unwrapKey(1L)); + + // Another KEK could unwrap a CBC blob to a wrong key without an error + verify(kmsKekVersionDao, never()).getVersionsForDecryption(anyLong()); + verify(kmsProvider).unwrapKey(any(WrappedKey.class), anyLong()); + } - HSMProfileVO profile = mock(HSMProfileVO.class); - when(profile.getProtocol()).thenReturn("database"); - when(profile.isEnabled()).thenReturn(true); - when(hsmProfileDao.findById(20L)).thenReturn(profile); + @Test + public void testUnwrapKey_FallsBackToOtherVersionsForAuthenticatedAlgorithm() { + mockWrappedKey(5L, "AES/GCM/NoPadding"); + mockDatabaseProfile(); + KMSKekVersionVO recordedVersion = mockVersion(5L, 1, KMSKekVersionVO.Status.Previous); + when(kmsKekVersionDao.findById(5L)).thenReturn(recordedVersion); + KMSKekVersionVO activeVersion = mockVersion(6L, 2, KMSKekVersionVO.Status.Active); + when(kmsKekVersionDao.getVersionsForDecryption(10L)).thenReturn(List.of(recordedVersion, activeVersion)); + when(kmsProvider.unwrapKey(any(WrappedKey.class), anyLong())) + .thenThrow(KMSException.wrapUnwrapFailed("Failed to unwrap key")) + .thenReturn(new byte[32]); + + byte[] result = kmsManager.unwrapKey(1L); + + assertEquals(32, result.length); + // The recorded version and then the active one; the recorded version isn't tried twice + verify(kmsProvider, times(2)).unwrapKey(any(WrappedKey.class), anyLong()); + } - when(kmsProvider.unwrapKey(any(WrappedKey.class), anyLong())).thenReturn(new byte[]{1, 2, 3}); - doReturn(kmsProvider).when(kmsManager).getKMSProvider("database"); + @Test + public void testUnwrapKey_ThrowsForCbcWithoutRecordedVersion() { + mockWrappedKey(null, "AES/CBC/PKCS5Padding"); - byte[] result = kmsManager.unwrapKey(wrappedKeyId); + KMSException e = assertThrows(KMSException.class, () -> kmsManager.unwrapKey(1L)); - assertNotNull(result); - verify(kmsKekVersionDao, never()).getVersionsForDecryption(kmsKeyId); + // Nothing could be tried, so the key counts as gone + assertEquals(KMSException.ErrorType.KEK_NOT_FOUND, e.getErrorType()); + verify(kmsKekVersionDao, never()).getVersionsForDecryption(anyLong()); } - @Test(expected = KMSException.class) - public void testUnwrapKey_FallsBackToAllVersionsWhenNoHint() { - KMSWrappedKeyVO wrappedVO = mock(KMSWrappedKeyVO.class); - when(wrappedVO.getKmsKeyId()).thenReturn(10L); - when(wrappedVO.getKekVersionId()).thenReturn(null); - when(kmsWrappedKeyDao.findById(1L)).thenReturn(wrappedVO); - when(kmsKeyDao.findById(10L)).thenReturn(mock(KMSKeyVO.class)); + @Test + public void testUnwrapKey_ThrowsKekNotFoundForArchivedVersion() { + mockWrappedKey(5L, "AES/GCM/NoPadding"); + KMSKekVersionVO version = mockVersion(5L, 1, KMSKekVersionVO.Status.Archived); + when(kmsKekVersionDao.findById(5L)).thenReturn(version); - kmsManager.unwrapKey(1L); + KMSException e = assertThrows(KMSException.class, () -> kmsManager.unwrapKey(1L)); + + // Its KEK was deleted, so no other version is tried either + assertEquals(KMSException.ErrorType.KEK_NOT_FOUND, e.getErrorType()); + verify(kmsKekVersionDao, never()).getVersionsForDecryption(anyLong()); + verify(kmsProvider, never()).unwrapKey(any(WrappedKey.class), anyLong()); } @Test(expected = KMSException.class) public void testUnwrapKey_ThrowsWhenAllVersionsFail() { + mockWrappedKey(null, "AES/GCM/NoPadding"); + when(kmsKekVersionDao.getVersionsForDecryption(10L)).thenReturn(List.of()); + + kmsManager.unwrapKey(1L); + } + + @Test + public void testDeleteKMSWrappedKey_KeepsKeyNeededBySnapshot() { + Volume volume = mockVolumeWithWrappedKey(); + SnapshotVO snapshot = mockSnapshot(Snapshot.State.BackedUp); + when(snapshotDao.searchByVolumes(List.of(1L))).thenReturn(List.of(snapshot)); + + assertFalse(kmsManager.deleteKMSWrappedKey(volume)); + + verify(kmsWrappedKeyDao, never()).remove(anyLong()); + } + + @Test + public void testDeleteKMSWrappedKey_RemovesKeyWhenOnlyDestroyedSnapshotsAreLeft() { + Volume volume = mockVolumeWithWrappedKey(); + SnapshotVO snapshot = mockSnapshot(Snapshot.State.Destroyed); + when(snapshotDao.searchByVolumes(List.of(1L))).thenReturn(List.of(snapshot)); + when(kmsWrappedKeyDao.remove(50L)).thenReturn(true); + + assertTrue(kmsManager.deleteKMSWrappedKey(volume)); + + verify(kmsWrappedKeyDao).remove(50L); + } + + /** + * Wrapped key 1 of KMS key 10 (volume encryption), recorded with the given KEK version and algorithm. + */ + private KMSWrappedKeyVO mockWrappedKey(Long kekVersionId, String wrapAlgorithm) { KMSWrappedKeyVO wrappedVO = mock(KMSWrappedKeyVO.class); when(wrappedVO.getKmsKeyId()).thenReturn(10L); - when(wrappedVO.getKekVersionId()).thenReturn(null); + when(wrappedVO.getKekVersionId()).thenReturn(kekVersionId); + Mockito.lenient().when(wrappedVO.getWrapAlgorithm()).thenReturn(wrapAlgorithm); + Mockito.lenient().when(wrappedVO.getWrappedBlob()).thenReturn(new byte[]{0, 1}); when(kmsWrappedKeyDao.findById(1L)).thenReturn(wrappedVO); - when(kmsKeyDao.findById(10L)).thenReturn(mock(KMSKeyVO.class)); - kmsManager.unwrapKey(1L); + KMSKeyVO kmsKey = mock(KMSKeyVO.class); + Mockito.lenient().when(kmsKey.getId()).thenReturn(10L); + Mockito.lenient().when(kmsKey.getPurpose()).thenReturn(KeyPurpose.VOLUME_ENCRYPTION); + when(kmsKeyDao.findById(10L)).thenReturn(kmsKey); + return wrappedVO; + } + + /** + * A KEK version of KMS key 10 in HSM profile 20 (see {@link #mockDatabaseProfile()}). + */ + private KMSKekVersionVO mockVersion(Long id, int versionNumber, KMSKekVersionVO.Status status) { + KMSKekVersionVO version = mock(KMSKekVersionVO.class); + Mockito.lenient().when(version.getId()).thenReturn(id); + Mockito.lenient().when(version.getVersionNumber()).thenReturn(versionNumber); + Mockito.lenient().when(version.getStatus()).thenReturn(status); + Mockito.lenient().when(version.getHsmProfileId()).thenReturn(20L); + Mockito.lenient().when(version.getKekLabel()).thenReturn("kek-label-" + versionNumber); + return version; + } + + /** + * HSM profile 20, an enabled profile of the "database" provider. + */ + private void mockDatabaseProfile() { + HSMProfileVO profile = mock(HSMProfileVO.class); + when(profile.getProtocol()).thenReturn("database"); + when(profile.isEnabled()).thenReturn(true); + when(hsmProfileDao.findById(20L)).thenReturn(profile); + doReturn(kmsProvider).when(kmsManager).getKMSProvider("database"); + } + + /** + * Volume 1, the only volume using wrapped key 50. + */ + private Volume mockVolumeWithWrappedKey() { + Volume volume = mock(Volume.class); + when(volume.getKmsWrappedKeyId()).thenReturn(50L); + KMSWrappedKeyVO wrappedKey = mock(KMSWrappedKeyVO.class); + when(wrappedKey.getId()).thenReturn(50L); + when(kmsWrappedKeyDao.findById(50L)).thenReturn(wrappedKey); + when(volumeDao.findByKmsWrappedKeyId(50L)).thenReturn(List.of()); + + VolumeVO volumeVO = mock(VolumeVO.class); + when(volumeVO.getId()).thenReturn(1L); + when(volumeDao.listIncludingRemovedByKmsWrappedKeyId(50L)).thenReturn(List.of(volumeVO)); + return volume; + } + + private SnapshotVO mockSnapshot(Snapshot.State state) { + SnapshotVO snapshot = mock(SnapshotVO.class); + when(snapshot.getState()).thenReturn(state); + return snapshot; } @Test(expected = KMSException.class) @@ -362,29 +489,49 @@ public void testDeleteUserKMSKey_DeletesKekFromProviderAndRemovesKey() throws KM } @Test - public void testDeleteUserKMSKey_ContinuesWhenKekDeletionFails() throws KMSException { - Long keyId = 1L; + public void testDeleteUserKMSKey_KeepsKeyWhenKekDeletionFails() throws KMSException { + KMSKeyVO key = mockDeletableKeyWithOneKekVersion(); + doThrow(KMSException.kekOperationFailed("provider error")).when(kmsProvider).deleteKek(anyString()); + + try (MockedStatic actionEventUtils = Mockito.mockStatic(ActionEventUtils.class)) { + assertThrows(CloudRuntimeException.class, () -> kmsManager.deleteUserKMSKey(key)); + + // The KEK is still in the HSM, so keep the key that refers to it + verify(kmsKeyDao, never()).remove(anyLong()); + } + } + + @Test + public void testDeleteUserKMSKey_ContinuesWhenKekIsAlreadyGone() throws KMSException { + KMSKeyVO key = mockDeletableKeyWithOneKekVersion(); + doThrow(KMSException.kekNotFound("Key with label 'kek-label' not found in HSM")).when(kmsProvider).deleteKek(anyString()); + + try (MockedStatic actionEventUtils = Mockito.mockStatic(ActionEventUtils.class)) { + kmsManager.deleteUserKMSKey(key); + + verify(kmsKeyDao).remove(1L); + } + } + + /** + * KMS key 1, not used by any wrapped key or volume, with one KEK version in a "database" HSM profile. + */ + private KMSKeyVO mockDeletableKeyWithOneKekVersion() { KMSKeyVO key = mock(KMSKeyVO.class); - when(key.getId()).thenReturn(keyId); - when(kmsWrappedKeyDao.countByKmsKeyId(keyId)).thenReturn(0L); - when(volumeDao.existsWithKmsKey(keyId)).thenReturn(false); + when(key.getId()).thenReturn(1L); + when(kmsWrappedKeyDao.countByKmsKeyId(1L)).thenReturn(0L); + when(volumeDao.existsWithKmsKey(1L)).thenReturn(false); KMSKekVersionVO version = mock(KMSKekVersionVO.class); when(version.getHsmProfileId()).thenReturn(10L); when(version.getKekLabel()).thenReturn("kek-label"); - when(kmsKekVersionDao.listByKmsKeyId(keyId)).thenReturn(List.of(version)); + when(kmsKekVersionDao.listByKmsKeyId(1L)).thenReturn(List.of(version)); HSMProfileVO profile = mock(HSMProfileVO.class); when(profile.getProtocol()).thenReturn("database"); when(hsmProfileDao.findById(10L)).thenReturn(profile); doReturn(kmsProvider).when(kmsManager).getKMSProvider("database"); - doThrow(KMSException.kekOperationFailed("provider error")).when(kmsProvider).deleteKek(anyString()); - - try (MockedStatic actionEventUtils = Mockito.mockStatic(ActionEventUtils.class)) { - kmsManager.deleteUserKMSKey(key); - - verify(kmsKeyDao).remove(keyId); - } + return key; } @Test diff --git a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyRotationTest.java b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyRotationTest.java index ed0df98e3951..6c208ed06f3c 100644 --- a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyRotationTest.java +++ b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplKeyRotationTest.java @@ -42,11 +42,14 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; @@ -137,6 +140,7 @@ public void testRotateKek_SameHSM() throws Exception { // Provider creates new KEK when(kmsProvider.createKek(any(KeyPurpose.class), eq(newKekLabel), anyInt(), eq(oldProfileId))) .thenReturn("new-kek-id"); + mockWrapRoundTrip(oldProfileId); KMSKekVersionVO newVersion = mock(KMSKekVersionVO.class); when(newVersion.getVersionNumber()).thenReturn(2); @@ -149,6 +153,8 @@ public void testRotateKek_SameHSM() throws Exception { // Verify new KEK was created in same HSM assertNotNull(result); verify(kmsProvider).createKek(any(KeyPurpose.class), eq(newKekLabel), eq(256), eq(oldProfileId)); + // The key shows the algorithm of its new KEK version + verify(kmsKey).setAlgorithm("AES/GCM/NoPadding"); // Verify old version marked as Previous verify(oldVersion).setStatus(KMSKekVersionVO.Status.Previous); @@ -192,6 +198,7 @@ public void testRotateKek_CrossHSMMigration() throws Exception { // Provider creates new KEK in different HSM when(kmsProvider.createKek(any(KeyPurpose.class), eq(newKekLabel), anyInt(), eq(newProfileId))) .thenReturn("new-kek-id"); + mockWrapRoundTrip(newProfileId); KMSKekVersionVO newVersion = mock(KMSKekVersionVO.class); when(newVersion.getVersionNumber()).thenReturn(2); @@ -247,6 +254,7 @@ public void testRewrapSingleKey_UnwrapAndRewrap() throws Exception { WrappedKey newWrappedKey = mock(WrappedKey.class); when(newWrappedKey.getWrappedKeyMaterial()).thenReturn("new-wrapped-blob".getBytes()); + when(newWrappedKey.getAlgorithm()).thenReturn("AES/GCM/NoPadding"); when(kmsProvider.wrapKey(plainDek, KeyPurpose.VOLUME_ENCRYPTION, "new-kek-label", newProfileId)) .thenReturn(newWrappedKey); @@ -262,6 +270,8 @@ public void testRewrapSingleKey_UnwrapAndRewrap() throws Exception { // Verify wrapped key was updated verify(wrappedKeyVO).setKekVersionId(newVersionId); verify(wrappedKeyVO).setWrappedBlob("new-wrapped-blob".getBytes()); + // The algorithm of the new blob is recorded with it + verify(wrappedKeyVO).setWrapAlgorithm("AES/GCM/NoPadding"); verify(kmsWrappedKeyDao).update(wrappedKeyId, wrappedKeyVO); } } @@ -298,6 +308,7 @@ public void testRotateKek_GeneratesLabel() throws Exception { // Provider creates new KEK - will accept any label when(kmsProvider.createKek(any(KeyPurpose.class), anyString(), anyInt(), eq(oldProfileId))) .thenReturn("new-kek-id"); + mockWrapRoundTrip(oldProfileId); KMSKekVersionVO newVersion = mock(KMSKekVersionVO.class); when(newVersion.getVersionNumber()).thenReturn(2); @@ -368,6 +379,7 @@ public void testRotateKek_UsesCurrentProfileWhenTargetNull() throws Exception { when(kmsProvider.createKek(any(KeyPurpose.class), anyString(), anyInt(), eq(currentProfileId))) .thenReturn("new-kek-id"); + mockWrapRoundTrip(currentProfileId); KMSKekVersionVO newVersion = mock(KMSKekVersionVO.class); when(newVersion.getVersionNumber()).thenReturn(2); @@ -384,4 +396,48 @@ public void testRotateKek_UsesCurrentProfileWhenTargetNull() throws Exception { // Verify KMS key was not updated (same profile) verify(kmsKey, never()).setHsmProfileId(currentProfileId); } + + /** + * Test: rotateKek deletes the new KEK and keeps the current version when the HSM can't wrap with it + */ + @Test + public void testRotateKek_DeletesNewKekWhenWrapTestFails() { + Long profileId = 10L; + + KMSKeyVO kmsKey = mock(KMSKeyVO.class); + when(kmsKey.getHsmProfileId()).thenReturn(profileId); + when(kmsKey.getPurpose()).thenReturn(KeyPurpose.VOLUME_ENCRYPTION); + + HSMProfileVO hsmProfile = mock(HSMProfileVO.class); + when(hsmProfile.getId()).thenReturn(profileId); + when(hsmProfile.getProtocol()).thenReturn(testProviderName); + when(hsmProfileDao.findById(profileId)).thenReturn(hsmProfile); + + when(kmsProvider.createKek(any(KeyPurpose.class), eq("new-label"), eq(256), eq(profileId))) + .thenReturn("new-kek-id"); + when(kmsProvider.getWrapAlgorithm(profileId)).thenReturn("AES/GCM/NoPadding"); + when(kmsProvider.wrapKey(any(byte[].class), any(KeyPurpose.class), eq("new-kek-id"), eq(profileId))) + .thenThrow(KMSException.kekOperationFailed("AES/GCM/NoPadding not supported by HSM")); + doReturn(kmsProvider).when(kmsManager).getKMSProvider(testProviderName); + + assertThrows(KMSException.class, () -> kmsManager.rotateKek(kmsKey, "old-label", "new-label", 256, null)); + + verify(kmsProvider).deleteKek("new-kek-id", profileId); + verify(kmsKekVersionDao, never()).persist(any(KMSKekVersionVO.class)); + verify(kmsKeyDao, never()).update(anyLong(), any(KMSKeyVO.class)); + } + + /** + * Makes the provider hand back whatever was wrapped, for the wrap test done after a KEK is created. + */ + private void mockWrapRoundTrip(Long hsmProfileId) { + when(kmsProvider.getWrapAlgorithm(hsmProfileId)).thenReturn("AES/GCM/NoPadding"); + AtomicReference wrapped = new AtomicReference<>(); + when(kmsProvider.wrapKey(any(byte[].class), any(KeyPurpose.class), anyString(), eq(hsmProfileId))) + .thenAnswer(invocation -> { + wrapped.set(((byte[]) invocation.getArgument(0)).clone()); + return mock(WrappedKey.class); + }); + when(kmsProvider.unwrapKey(any(WrappedKey.class), eq(hsmProfileId))).thenAnswer(invocation -> wrapped.get().clone()); + } } diff --git a/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplMigrationTest.java b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplMigrationTest.java new file mode 100644 index 000000000000..38f2c86c500f --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/kms/KMSManagerImplMigrationTest.java @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.kms; + +import com.cloud.event.ActionEventUtils; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.db.GlobalLock; +import org.apache.cloudstack.framework.kms.KMSException; +import org.apache.cloudstack.framework.kms.KMSProvider; +import org.apache.cloudstack.framework.kms.KeyPurpose; +import org.apache.cloudstack.framework.kms.WrappedKey; +import org.apache.cloudstack.kms.dao.HSMProfileDao; +import org.apache.cloudstack.kms.dao.KMSKekVersionDao; +import org.apache.cloudstack.kms.dao.KMSKeyDao; +import org.apache.cloudstack.kms.dao.KMSWrappedKeyDao; +import org.apache.cloudstack.secret.PassphraseVO; +import org.apache.cloudstack.secret.dao.PassphraseDao; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for migrating legacy volume passphrases to KMS, and for repairing the keys 4.23 migrated. + */ +@RunWith(MockitoJUnitRunner.class) +public class KMSManagerImplMigrationTest { + + private static final Long PASSPHRASE_ID = 7L; + private static final Long WRAPPED_KEY_ID = 5L; + private static final Long KMS_KEY_ID = 10L; + private static final Long HSM_PROFILE_ID = 20L; + private static final Long KEK_VERSION_ID = 30L; + private static final String KEK_LABEL = "kek-label"; + + @Spy + @InjectMocks + private KMSManagerImpl kmsManager; + + @Mock + private KMSKeyDao kmsKeyDao; + + @Mock + private KMSKekVersionDao kmsKekVersionDao; + + @Mock + private KMSWrappedKeyDao kmsWrappedKeyDao; + + @Mock + private HSMProfileDao hsmProfileDao; + + @Mock + private VolumeDao volumeDao; + + @Mock + private PassphraseDao passphraseDao; + + @Mock + private KMSProvider kmsProvider; + + private MockedStatic actionEventUtils; + + @Before + public void setUp() { + actionEventUtils = Mockito.mockStatic(ActionEventUtils.class); + } + + @After + public void tearDown() { + actionEventUtils.close(); + } + + @Test + public void testMigrateVolumeToKmsKey_WrapsRawPassphraseAndRemovesLegacyRow() { + byte[] passphrase = legacyPassphrase(); + VolumeVO volume = mockLegacyVolume(passphrase); + AtomicReference wrappedPlainKey = mockWrap(); + when(kmsWrappedKeyDao.persist(any(KMSWrappedKeyVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + when(volumeDao.listIncludingRemovedByPassphraseId(PASSPHRASE_ID)).thenReturn(List.of()); + + assertTrue(kmsManager.migrateVolumeToKmsKey(kmsProvider, volume, mockKmsKey(), mockKekVersion())); + + // The raw bytes, so that the base64 text handed to qemu stays the same passphrase + assertArrayEquals(passphrase, wrappedPlainKey.get()); + ArgumentCaptor wrappedKeyCaptor = ArgumentCaptor.forClass(KMSWrappedKeyVO.class); + verify(kmsWrappedKeyDao).persist(wrappedKeyCaptor.capture()); + assertEquals("AES/GCM/NoPadding", wrappedKeyCaptor.getValue().getWrapAlgorithm()); + verify(volume).setPassphraseId(null); + // The volume key isn't left behind in the passphrase table + verify(passphraseDao).remove(PASSPHRASE_ID); + } + + @Test + public void testMigrateVolumeToKmsKey_KeepsLegacyRowUsedByAnotherVolume() { + VolumeVO volume = mockLegacyVolume(legacyPassphrase()); + mockWrap(); + when(kmsWrappedKeyDao.persist(any(KMSWrappedKeyVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + when(volumeDao.listIncludingRemovedByPassphraseId(PASSPHRASE_ID)).thenReturn(List.of(mock(VolumeVO.class))); + + assertTrue(kmsManager.migrateVolumeToKmsKey(kmsProvider, volume, mockKmsKey(), mockKekVersion())); + + verify(passphraseDao, never()).remove(anyLong()); + } + + @Test + public void testRepairKeyFormatBatch_RewrapsKeyMigratedBy423() { + byte[] passphrase = legacyPassphrase(); + KMSWrappedKeyVO wrappedKey = flaggedWrappedKey(); + mockActiveVersionInDatabaseProfile(); + // 4.23 wrapped the base64 text of the passphrase + doReturn(Base64.getEncoder().encode(passphrase)).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + KMSKeyVO kmsKey = mock(KMSKeyVO.class); + when(kmsKey.getId()).thenReturn(KMS_KEY_ID); + when(kmsKey.getPurpose()).thenReturn(KeyPurpose.VOLUME_ENCRYPTION); + when(kmsKeyDao.findById(KMS_KEY_ID)).thenReturn(kmsKey); + AtomicReference wrappedPlainKey = mockWrap(); + VolumeVO volume = mockVolumeCreated(new Date(0)); + when(volumeDao.findByKmsWrappedKeyId(WRAPPED_KEY_ID)).thenReturn(List.of(volume)); + + assertFalse(runRepairKeyFormatBatch()); + + assertArrayEquals(passphrase, wrappedPlainKey.get()); + assertArrayEquals(new byte[]{1, 2, 3}, wrappedKey.getWrappedBlob()); + assertEquals("AES/GCM/NoPadding", wrappedKey.getWrapAlgorithm()); + assertFalse(wrappedKey.isFormatCheckPending()); + verify(kmsWrappedKeyDao).update(WRAPPED_KEY_ID, wrappedKey); + } + + @Test + public void testRepairKeyFormatBatch_OnlyClearsFlagOfNativeKey() { + KMSWrappedKeyVO wrappedKey = flaggedWrappedKey(); + doReturn(new byte[32]).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + + assertFalse(runRepairKeyFormatBatch()); + + assertArrayEquals(new byte[]{9}, wrappedKey.getWrappedBlob()); + assertFalse(wrappedKey.isFormatCheckPending()); + verify(kmsWrappedKeyDao).update(WRAPPED_KEY_ID, wrappedKey); + verify(kmsProvider, never()).wrapKey(any(byte[].class), any(KeyPurpose.class), any(), any()); + } + + @Test + public void testRepairKeyFormatBatch_KeepsFlagWhenKeyCantBeUnwrapped() { + KMSWrappedKeyVO wrappedKey = flaggedWrappedKey(); + doThrow(KMSException.wrapUnwrapFailed("HSM unreachable")).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + + assertFalse(runRepairKeyFormatBatch()); + + // Retried in a later run + assertTrue(wrappedKey.isFormatCheckPending()); + verify(kmsWrappedKeyDao, never()).update(anyLong(), any(KMSWrappedKeyVO.class)); + } + + @Test + public void testRepairKeyFormatBatch_SkipsKeyUsedByVolumeCreatedAfterMigration() { + KMSWrappedKeyVO wrappedKey = flaggedWrappedKey(); + doReturn(Base64.getEncoder().encode(legacyPassphrase())).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + // e.g. the copy made by a storage migration on 4.23, encrypted with the passphrase 4.23 used + VolumeVO volume = mockVolumeCreated(new Date(wrappedKey.getCreated().getTime() + 60_000)); + when(volumeDao.findByKmsWrappedKeyId(WRAPPED_KEY_ID)).thenReturn(List.of(volume)); + + assertFalse(runRepairKeyFormatBatch()); + + assertArrayEquals(new byte[]{9}, wrappedKey.getWrappedBlob()); + assertFalse(wrappedKey.isFormatCheckPending()); + verify(kmsProvider, never()).wrapKey(any(byte[].class), any(KeyPurpose.class), any(), any()); + } + + @Test + public void testRepairKeyFormatBatch_ClearsFlagOfKeyThatIsGone() { + KMSWrappedKeyVO wrappedKey = flaggedWrappedKey(); + doThrow(KMSException.kekNotFound("KMS key not found for wrapped key: 5")).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + + assertFalse(runRepairKeyFormatBatch()); + + // Nothing left to repair, so the repair doesn't keep retrying it + assertFalse(wrappedKey.isFormatCheckPending()); + verify(kmsWrappedKeyDao).update(WRAPPED_KEY_ID, wrappedKey); + } + + @Test + public void testRepairKeyFormatBatch_StopsAtFirstFailure() { + KMSWrappedKeyVO failingKey = new KMSWrappedKeyVO(KMS_KEY_ID, KEK_VERSION_ID, 1L, new byte[]{9}); + failingKey.setId(WRAPPED_KEY_ID); + failingKey.setFormatCheckPending(true); + KMSWrappedKeyVO nextKey = new KMSWrappedKeyVO(KMS_KEY_ID, KEK_VERSION_ID, 1L, new byte[]{9}); + nextKey.setId(6L); + nextKey.setFormatCheckPending(true); + when(kmsWrappedKeyDao.listFormatCheckPending(eq(0L), anyInt())).thenReturn(List.of(failingKey, nextKey)); + doThrow(KMSException.wrapUnwrapFailed("HSM unreachable")).when(kmsManager).unwrapKey(WRAPPED_KEY_ID); + + assertFalse(runRepairKeyFormatBatch()); + + // The lock it shares with the rewrap worker isn't held through more failing unwraps + verify(kmsManager, never()).unwrapKey(6L); + } + + @Test + public void testRepairKeyFormatBatch_DoneWhenNoKeyIsFlagged() { + when(kmsWrappedKeyDao.listFormatCheckPending(eq(0L), anyInt())).thenReturn(List.of()); + + assertTrue(runRepairKeyFormatBatch()); + } + + @Test + public void testDecodeLegacyMigratedKey() { + byte[] passphrase = legacyPassphrase(); + assertArrayEquals(passphrase, KMSManagerImpl.decodeLegacyMigratedKey(Base64.getEncoder().encode(passphrase))); + // A DEK created by KMS + assertNull(KMSManagerImpl.decodeLegacyMigratedKey(new byte[32])); + // 64 bytes that aren't base64 text + assertNull(KMSManagerImpl.decodeLegacyMigratedKey(new byte[64])); + } + + private boolean runRepairKeyFormatBatch() { + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + GlobalLock lock = mock(GlobalLock.class); + globalLock.when(() -> GlobalLock.getInternLock("kms.rewrap.worker")).thenReturn(lock); + when(lock.lock(5)).thenReturn(true); + return kmsManager.repairKeyFormatBatch(); + } + } + + private static byte[] legacyPassphrase() { + byte[] passphrase = new byte[48]; + for (int i = 0; i < passphrase.length; i++) { + passphrase[i] = (byte) (i * 7); + } + return passphrase; + } + + /** + * A legacy encrypted volume whose passphrase row holds the base64 text of the given passphrase. + */ + private VolumeVO mockLegacyVolume(byte[] passphrase) { + PassphraseVO passphraseVO = mock(PassphraseVO.class); + when(passphraseVO.getPassphraseString()).thenReturn(Base64.getEncoder().encodeToString(passphrase)); + when(passphraseDao.findById(PASSPHRASE_ID)).thenReturn(passphraseVO); + + VolumeVO volume = mock(VolumeVO.class); + when(volume.getAccountId()).thenReturn(2L); + when(volume.getPassphraseId()).thenReturn(PASSPHRASE_ID); + return volume; + } + + private VolumeVO mockVolumeCreated(Date created) { + VolumeVO volume = mock(VolumeVO.class); + when(volume.getCreated()).thenReturn(created); + return volume; + } + + private KMSKey mockKmsKey() { + KMSKey kmsKey = mock(KMSKey.class); + when(kmsKey.getAccountId()).thenReturn(2L); + when(kmsKey.getId()).thenReturn(KMS_KEY_ID); + return kmsKey; + } + + private KMSKekVersionVO mockKekVersion() { + KMSKekVersionVO version = mock(KMSKekVersionVO.class); + Mockito.lenient().when(version.getId()).thenReturn(KEK_VERSION_ID); + Mockito.lenient().when(version.getStatus()).thenReturn(KMSKekVersionVO.Status.Active); + Mockito.lenient().when(version.getKekLabel()).thenReturn(KEK_LABEL); + Mockito.lenient().when(version.getHsmProfileId()).thenReturn(HSM_PROFILE_ID); + return version; + } + + private void mockActiveVersionInDatabaseProfile() { + KMSKekVersionVO version = mockKekVersion(); + when(kmsKekVersionDao.getActiveVersion(KMS_KEY_ID)).thenReturn(version); + HSMProfileVO profile = mock(HSMProfileVO.class); + when(profile.getProtocol()).thenReturn("database"); + when(hsmProfileDao.findById(HSM_PROFILE_ID)).thenReturn(profile); + doReturn(kmsProvider).when(kmsManager).getKMSProvider("database"); + } + + /** + * Wrapped key 5, existing at the upgrade from 4.23 and not checked yet. + */ + private KMSWrappedKeyVO flaggedWrappedKey() { + KMSWrappedKeyVO wrappedKey = new KMSWrappedKeyVO(KMS_KEY_ID, KEK_VERSION_ID, 1L, new byte[]{9}); + wrappedKey.setId(WRAPPED_KEY_ID); + wrappedKey.setWrapAlgorithm("AES/CBC/PKCS5Padding"); + wrappedKey.setFormatCheckPending(true); + when(kmsWrappedKeyDao.listFormatCheckPending(eq(0L), anyInt())).thenReturn(List.of(wrappedKey)); + return wrappedKey; + } + + /** + * Makes the provider wrap with AES/GCM/NoPadding, recording the plain key it got. + */ + private AtomicReference mockWrap() { + AtomicReference plainKey = new AtomicReference<>(); + WrappedKey wrappedKey = mock(WrappedKey.class); + when(wrappedKey.getWrappedKeyMaterial()).thenReturn(new byte[]{1, 2, 3}); + when(wrappedKey.getAlgorithm()).thenReturn("AES/GCM/NoPadding"); + when(kmsProvider.wrapKey(any(byte[].class), eq(KeyPurpose.VOLUME_ENCRYPTION), eq(KEK_LABEL), eq(HSM_PROFILE_ID))) + .thenAnswer(invocation -> { + plainKey.set(((byte[]) invocation.getArgument(0)).clone()); + return wrappedKey; + }); + return plainKey; + } +} diff --git a/ui/src/config/section/kms.js b/ui/src/config/section/kms.js index 3120fee54cc8..0f0681479554 100644 --- a/ui/src/config/section/kms.js +++ b/ui/src/config/section/kms.js @@ -237,7 +237,7 @@ export default { }, mapping: { details: { - optionalKeys: ['pin', 'library', 'slot', 'slot_list_index', 'token_label'] + optionalKeys: ['pin', 'library', 'slot', 'slot_list_index', 'wrap_algorithm'] } } },