diff --git a/api/src/main/java/com/cloud/agent/api/to/FilesystemInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/FilesystemInfoTO.java new file mode 100644 index 000000000000..9e938dd19c01 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/FilesystemInfoTO.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api.to; + +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class FilesystemInfoTO { + private final String name; + private final String filesystem; + private final long size; + private final long volumeId; + + public FilesystemInfoTO(String name, String filesystem, long size, long volumeId) { + this.name = name; + this.filesystem = filesystem; + this.size = size; + this.volumeId = volumeId; + } + + public String getName() { + return name; + } + + public String getFilesystem() { + return filesystem; + } + + public long getSize() { + return size; + } + + public long getVolumeId() { + return volumeId; + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f7d13343d469..02a75b0cf191 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -669,6 +669,7 @@ public class EventTypes { public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT"; public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP"; public static final String EVENT_SCREENSHOT_DOWNLOAD = "BACKUP.VALIDATION.SCREENSHOT.DOWNLOAD"; + public static final String EVENT_BACKUP_FILE_DOWNLOAD = "BACKUP.FILE.DOWNLOAD"; // external network device events public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD"; diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 3511b4e88cb9..c8abf19a7d83 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -36,7 +36,9 @@ public static enum ImageFormat { TAR(false, false, false, "tar"), ZIP(false, false, false, "zip"), DIR(false, false, false, "dir"), - PNG(false, false, false, "png"); + PNG(false, false, false, "png"), + GZIP(false, false, false, "gz"), + TARGZ(false, false, false, "tar.gz"); private final boolean supportThinProvisioning; private final boolean supportSparse; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiArgValidator.java b/api/src/main/java/org/apache/cloudstack/api/ApiArgValidator.java index 380472352735..b680565ebe4f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiArgValidator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiArgValidator.java @@ -37,4 +37,10 @@ public enum ApiArgValidator { * Validates if the parameter is a valid RFC Compliance domain name. */ RFCComplianceDomainName, + + /** + * Validates that the parameter does not match the following regex '[$|&*`\@!%'"^;<>!()]' + * Some special characters are allowed, such as '/'. If you need to disallow all special characters, a new validator should be created. + * */ + LimitedSpecialCharacters, } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ac6acdf42516..53add0c8b197 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1489,6 +1489,10 @@ public class ApiConstants { public static final String SCHEDULED = "scheduled"; public static final String SCHEDULED_DATE = "scheduleddate"; public static final String BACKUP_PROVIDER = "backupprovider"; + public static final String IS_FILESYSTEM = "isfilesystem"; + public static final String IS_SYMLINK = "issymlink"; + public static final String BROWSABLE = "browsable"; + public static final String CANONICAL_PATH = "canonicalpath"; /** * This enum specifies IO Drivers, each option controls specific policies on I/O. diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DownloadBackupFileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DownloadBackupFileCmd.java new file mode 100644 index 000000000000..c80c134440ec --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DownloadBackupFileCmd.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; + +import javax.inject.Inject; + +@APICommand(name = "downloadBackupFile", + description = "Download a file from a backup", + responseObject = ExtractResponse.class, since = "4.24.0.0") +public class DownloadBackupFileCmd extends BaseAsyncCmd { + + @Inject + private BackupManager backupManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL + @Parameter(name = ApiConstants.BACKUP_ID, type = BaseCmd.CommandType.UUID, entityType = BackupResponse.class, required = true, + description = "ID of the backup to download the file.") + private Long backupId; + + @ACL + @Parameter(name = ApiConstants.VOLUME_ID, type = BaseCmd.CommandType.UUID, entityType = VolumeResponse.class, required = true, + description = "ID of the volume. If not informed, we will return every filesystem of every volume.") + private Long volumeId; + + @Parameter(name = ApiConstants.FILESYSTEM, type = BaseCmd.CommandType.STRING, required = true, + description = "Filesystem to list the files in.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String filesystem; + + @Parameter(name = ApiConstants.PATH, type = BaseCmd.CommandType.STRING, required = true, + description = "Path to the file to be downloaded in the backed-up volume.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String path; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public String getFilesystem() { + return filesystem.trim(); + } + + public String getPath() { + return path.trim(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation ////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return EventTypes.EVENT_BACKUP_FILE_DOWNLOAD; + } + + @Override + public String getEventDescription() { + Backup backup = _entityMgr.findById(Backup.class, getBackupId()); + if (backup == null) { + throw new InvalidParameterValueException(String.format("Unable to find backup with ID [%s].", getBackupId())); + } + return "Downloading a file from backup " + backup.getUuid(); + } + + @Override + public long getEntityOwnerId() { + Backup backup = _entityMgr.findById(Backup.class, getBackupId()); + if (backup != null) { + return backup.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + ExtractResponse response = backupManager.downloadBackupFile(getBackupId(), getVolumeId(), getFilesystem(), getPath()); + + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesCmd.java new file mode 100644 index 000000000000..f3ee50461275 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesCmd.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "listBackupFiles", + description = "List a backup inner files", + responseObject = DataStoreObjectResponse.class, since = "4.24.0.0") +public class ListBackupFilesCmd extends BaseListCmd { + + @Inject + private BackupManager backupManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL + @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, required = true, + description = "ID of the backup to list the files.") + private Long backupId; + + @ACL + @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, required = true, + description = "ID of the volume.") + private Long volumeId; + + @Parameter(name = ApiConstants.FILESYSTEM, type = CommandType.STRING, required = true, + description = "Filesystem to list the files in.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String filesystem; + + @Parameter(name = ApiConstants.PATH, type = CommandType.STRING, required = true, + description = "Path to list files in the backed-up volume.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String path; + + @Parameter(name = ApiConstants.IS_SYMLINK, type = CommandType.BOOLEAN, + description = "Path to list files in the backed-up volume.") + private Boolean isSymlink; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public String getFilesystem() { + return filesystem.trim(); + } + + public String getPath() { + return path.trim(); + } + + public Boolean getSymlink() { + return isSymlink; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + Backup backup = _entityMgr.findById(Backup.class, getBackupId()); + if (backup != null) { + return backup.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + List responseList = backupManager.listBackupFiles(getBackupId(), getVolumeId(), getFilesystem(), getPath(), getSymlink()); + + ListResponse response = new ListResponse<>(); + response.setResponses(responseList); + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesystemsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesystemsCmd.java new file mode 100644 index 000000000000..f2cfc8def7c1 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesystemsCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "listBackupFilesystems", + description = "List a backup inner file systems", + responseObject = DataStoreObjectResponse.class, since = "4.24.0.0") +public class ListBackupFilesystemsCmd extends BaseListCmd { + + @Inject + private BackupManager backupManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL + @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, required = true, + description = "ID of the backup to list the file systems.") + private Long backupId; + + @ACL + @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, + description = "ID of the volume. If not informed, we will return every filesystem of every volume.") + private Long volumeId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + Backup backup = _entityMgr.findById(Backup.class, getBackupId()); + if (backup != null) { + return backup.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + List responseList = backupManager.listBackupFilesystems(getBackupId(), getVolumeId()); + + ListResponse response = new ListResponse<>(); + response.setResponses(responseList); + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index 70db01445edd..f41e66b0ef55 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -151,6 +151,10 @@ public class BackupResponse extends BaseResponse { @Param(description = "Host ID where the backup is running", since = "4.23.0") private String hostId; + @SerializedName(ApiConstants.BROWSABLE) + @Param(description = "Whether it is possible to browse the backup files or not.") + private Boolean browsable; + public String getId() { return id; } @@ -386,4 +390,12 @@ public void setHostId(String hostId) { public String getHostId() { return this.hostId; } + + public Boolean getBrowsable() { + return browsable; + } + + public void setBrowsable(Boolean browsable) { + this.browsable = browsable; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 30fc2bbec40d..b79eb7e99000 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupOfferingCmd; import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ExtractResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -44,6 +45,7 @@ import com.cloud.utils.component.Manager; import com.cloud.utils.component.PluggableService; import com.cloud.vm.VmDiskInfo; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; /** * Backup and Recover Manager Interface @@ -243,6 +245,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException; + List listBackupFiles(long backupId, Long volumeId, String filesystem, String directory, Boolean isSymlink); + + ExtractResponse downloadBackupFile(long backupId, Long volumeId, String filesystem, String file); + void validateBackupForZone(Long zoneId); BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupOfferingCmd); @@ -264,4 +270,6 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Capacity getBackupStorageUsedStats(Long zoneId); void checkAndRemoveBackupOfferingBeforeExpunge(VirtualMachine vm); + + List listBackupFilesystems(long backupId, Long volumeId); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 66e4501c460e..8771e3077afe 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -18,8 +18,10 @@ import java.util.List; +import com.cloud.agent.api.to.FilesystemInfoTO; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; public interface BackupProvider { @@ -157,4 +159,19 @@ default boolean supportsMemoryVmSnapshot() { * @param zoneId the zone for which to return metrics */ void syncBackupStorageStats(Long zoneId); + + /** + * Returns a list of file system infos for the given backup. If the volume is not null, it should return a list of file systems specific to the volume. + * */ + default List listBackupFilesystems(Backup backup, Backup.VolumeInfo volumeInfo) { + return List.of(); + } + + default List listBackupFiles(Backup backup, Backup.VolumeInfo volumeInfo, String filesystem, String directory, Boolean isSymlink) { + return List.of(); + } + + default String downloadBackupFile(Backup backup, Backup.VolumeInfo volumeInfo, String filesystem, String file) { + return ""; + } } diff --git a/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java index c281fa115fdd..d6e39718631c 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java @@ -29,12 +29,24 @@ public class DataStoreObjectResponse extends BaseResponse { @Param(description = "Name of the data store object.") private String name; + @SerializedName(ApiConstants.CANONICAL_PATH) + @Param(description = "Canonical path of the object.") + private String canonicalPath; + @SerializedName("isdirectory") - @Param(description = "Is it a directory.") + @Param(description = "Indicates whether the path references a directory.") private boolean isDirectory; + @SerializedName(ApiConstants.IS_SYMLINK) + @Param(description = "Indicates whether the path references a symlink.") + private Boolean isSymlink; + + @SerializedName(ApiConstants.IS_FILESYSTEM) + @Param(description = "Indicates whether the path references a filesystem.") + private Boolean isFilesystem; + @SerializedName(ApiConstants.SIZE) - @Param(description = "Size is in Bytes.") + @Param(description = "Size in Bytes.") private long size; @SerializedName(ApiConstants.TEMPLATE_ID) @@ -46,7 +58,7 @@ public class DataStoreObjectResponse extends BaseResponse { private String templateName; @SerializedName(ApiConstants.FORMAT) - @Param(description = "Format of template associated with the data store object.") + @Param(description = "Format of object associated with the data store object.") private String format; @SerializedName(ApiConstants.SNAPSHOT_ID) @@ -110,6 +122,30 @@ public void setVolumeName(String volumeName) { this.volumeName = volumeName; } + public void setIsFilesystem(boolean isFilesystem) { + this.isFilesystem = isFilesystem; + } + + public void setIsSymlink(Boolean isSymlink) { + this.isSymlink = isSymlink; + } + + public void setCanonicalPath(String canonicalPath) { + this.canonicalPath = canonicalPath; + } + + public boolean isFilesystem() { + return isFilesystem; + } + + public Boolean isSymlink() { + return isSymlink; + } + + public String getCanonicalPath() { + return canonicalPath; + } + public String getName() { return name; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/ExtractBackupFileCommand.java b/core/src/main/java/org/apache/cloudstack/backup/ExtractBackupFileCommand.java new file mode 100644 index 000000000000..7c31468a9fe7 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/ExtractBackupFileCommand.java @@ -0,0 +1,91 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.NfsTO; + +import java.util.HashSet; +import java.util.Set; + +public class ExtractBackupFileCommand extends Command { + + private long volumeId; + private String filePath; + private String destinationPath; + private String filesystem; + private String backupPath; + private NfsTO datastore; + private Set secondaryStorageUrls; + private boolean cleanup; + + public ExtractBackupFileCommand(long volumeId, String filesystem, String filePath, String destinationPath, String backupPath, NfsTO datastore, Set secondaryStorageUrls) { + this.volumeId = volumeId; + this.filePath = filePath; + this.destinationPath = destinationPath; + this.filesystem = filesystem; + this.backupPath = backupPath; + this.datastore = datastore; + this.secondaryStorageUrls = secondaryStorageUrls; + } + + public ExtractBackupFileCommand(String destinationPath, NfsTO datastore) { + this.cleanup = true; + this.destinationPath = destinationPath; + this.datastore = datastore; + this.secondaryStorageUrls = new HashSet<>(); + } + + public long getVolumeId() { + return volumeId; + } + + public String getFilePath() { + return filePath; + } + + public String getDestinationPath() { + return destinationPath; + } + + public String getFilesystem() { + return filesystem; + } + + public String getBackupPath() { + return backupPath; + } + + public NfsTO getDatastore() { + return datastore; + } + + public Set getSecondaryStorageUrls() { + return secondaryStorageUrls; + } + + public boolean isCleanup() { + return cleanup; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/ListFilesCommand.java b/core/src/main/java/org/apache/cloudstack/backup/ListFilesCommand.java new file mode 100644 index 000000000000..c05f3c673c69 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/ListFilesCommand.java @@ -0,0 +1,78 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.NfsTO; + +import java.util.Set; + +public class ListFilesCommand extends Command { + + private long volumeId; + private String dirPath; + private String filesystem; + private String backupPath; + private Boolean isSymlink; + private NfsTO datastore; + private Set secondaryStorageUrls; + + public ListFilesCommand(long volumeId, String filesystem, String dirPath, String backupPath, NfsTO datastore, Boolean isSymlink, Set secondaryStorageUrls) { + this.volumeId = volumeId; + this.filesystem = filesystem; + this.dirPath = dirPath; + this.backupPath = backupPath; + this.isSymlink = isSymlink; + this.datastore = datastore; + this.secondaryStorageUrls = secondaryStorageUrls; + } + + public long getVolumeId() { + return volumeId; + } + + public String getDirPath() { + return dirPath; + } + + public String getFilesystem() { + return filesystem; + } + + public String getBackupPath() { + return backupPath; + } + + public Boolean isSymlink() { + return isSymlink; + } + + public NfsTO getDatastore() { + return datastore; + } + + public Set getSecondaryStorageUrls() { + return secondaryStorageUrls; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsAnswer.java new file mode 100644 index 000000000000..cd15dd0ef6ac --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsAnswer.java @@ -0,0 +1,38 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.FilesystemInfoTO; + +import java.util.List; + +public class ListFilesystemsAnswer extends Answer { + List filesystemInfoTOList; + + public ListFilesystemsAnswer(Command command, List filesystemInfoTOList) { + super(command); + this.filesystemInfoTOList = filesystemInfoTOList; + } + + public List getFilesystemInfoTOList() { + return filesystemInfoTOList; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsCommand.java b/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsCommand.java new file mode 100644 index 000000000000..a7b2b7a6786a --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/ListFilesystemsCommand.java @@ -0,0 +1,55 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.NfsTO; + +import java.util.HashMap; +import java.util.Set; + +public class ListFilesystemsCommand extends Command { + + private HashMap volumeIdAndBackupPath; + private NfsTO datastore; + private Set secondaryStorageUrls; + + public ListFilesystemsCommand(HashMap volumeIdAndBackupPath, NfsTO datastore, Set secondaryStorageUrls) { + this.volumeIdAndBackupPath = volumeIdAndBackupPath; + this.datastore = datastore; + this.secondaryStorageUrls = secondaryStorageUrls; + } + + public HashMap getVolumeIdAndBackupPath() { + return volumeIdAndBackupPath; + } + + public NfsTO getDatastore() { + return datastore; + } + + public Set getSecondaryStorageUrls() { + return secondaryStorageUrls; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java index eb8d0991c5c7..2f2beb50d8cb 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java @@ -38,6 +38,8 @@ public class ListDataStoreObjectsAnswer extends Answer { private List isDirs; + private List isSymlinks; + private List sizes; private List lastModified; @@ -61,6 +63,20 @@ public ListDataStoreObjectsAnswer(boolean pathExists, int count, List na this.lastModified = lastModified; } + public ListDataStoreObjectsAnswer(boolean pathExists, int count, List names, List paths, List absPaths, List isDirs, List isSymlinks, + List sizes, List lastModified) { + super(); + this.pathExists = pathExists; + this.count = count; + this.names = names; + this.paths = paths; + this.absPaths = absPaths; + this.isDirs = isDirs; + this.isSymlinks = isSymlinks; + this.sizes = sizes; + this.lastModified = lastModified; + } + public boolean isPathExists() { return pathExists; } @@ -97,6 +113,13 @@ public List getIsDirs() { return isDirs; } + public List getIsSymlinks() { + if (isSymlinks == null) { + return Collections.emptyList(); + } + return isSymlinks; + } + public List getSizes() { if (sizes == null) { return Collections.emptyList(); diff --git a/debian/control b/debian/control index cdf663ef8906..83efb6cf2e1a 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat, guestfish, guestfs-tools Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3ba2e4d5789e..bd84690f3a86 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -129,6 +129,7 @@ Requires: (selinux-tools if selinux-tools) Requires: sysstat Requires: python3-libnbd Requires: socat +Requires: guestfs-tools Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java index 596dc62b7206..9ce2c655d5c0 100644 --- a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java +++ b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java @@ -55,6 +55,8 @@ import org.apache.cloudstack.backup.dao.InternalBackupJoinDao; import org.apache.cloudstack.backup.dao.InternalBackupServiceJobDao; import org.apache.cloudstack.backup.dao.InternalBackupStoragePoolDao; +import org.apache.cloudstack.backup.to.BackupFileObject; +import org.apache.cloudstack.backup.to.BackupFileTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -74,15 +76,20 @@ import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; import org.apache.cloudstack.storage.command.BackupDeleteAnswer; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.BackupDeltaTO; import org.apache.cloudstack.storage.to.DeltaMergeTreeTO; import org.apache.cloudstack.storage.to.KbossTO; @@ -101,6 +108,8 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; import com.cloud.exception.AgentUnavailableException; @@ -272,6 +281,9 @@ public class KbossBackupProvider extends AdapterBase implements InternalBackupPr @Inject private AlertManager alertManager; + @Inject + private ImageStoreObjectDownloadDao imageStoreObjectDownloadDao; + protected final List validChildStatesToRemoveBackup = List.of(Backup.Status.Expunged, Backup.Status.Error, Backup.Status.Failed); private final List supportedStoragePoolTypes = List.of(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem, @@ -1013,6 +1025,141 @@ public Set getSecondaryStorageUrls(UserVm userVm) { return secondaryStorageUrls; } + @Override + public List listBackupFilesystems(Backup backup, Backup.VolumeInfo volumeInfo) { + List volumeInfos = new ArrayList<>(); + if (volumeInfo != null) { + volumeInfos.add(volumeInfo); + } else { + volumeInfos = backup.getBackedUpVolumes(); + } + + HashMap volumeIdToVolumeBackupPath = new HashMap<>(); + for (Backup.VolumeInfo info : volumeInfos) { + VolumeVO volumeVO = volumeDao.findByUuidIncludingRemoved(info.getUuid()); + InternalBackupDataStoreVO internalBackupDataStoreVO = internalBackupDataStoreDao.findByBackupIdAndVolumeId(backup.getId(), volumeVO.getId()); + volumeIdToVolumeBackupPath.put(volumeVO.getId(), internalBackupDataStoreVO.getBackupPath()); + } + + InternalBackupJoinVO internalBackupJoinVO = internalBackupJoinDao.findById(backup.getId()); + DataStore dataStore = dataStoreManager.getDataStore(internalBackupJoinVO.getImageStoreId(), DataStoreRole.Image); + DataStoreTO dataStoreTO = dataStore.getTO(); + Set secondaryStorageUrls = getParentSecondaryStorageUrls((BackupVO)backup); + ListFilesystemsCommand cmd = new ListFilesystemsCommand(volumeIdToVolumeBackupPath, (NfsTO) dataStoreTO, secondaryStorageUrls); + + String errorMessage = String.format("Unable to list filesystems of backup [%s]. Please check the logs.", backup.getUuid()); + EndPoint endPoint = getRandomEndpointToListBackupFilesOrFilesystems(backup, errorMessage, "filesystems"); + + Answer answer = endPoint.sendMessage(cmd); + if (answer == null || !answer.getResult()) { + logger.error("Unable to list filesystems of backup [{}] due to [{}].", backup, answer == null ? "null answer" : answer.getDetails()); + throw new CloudRuntimeException(errorMessage); + } + + ListFilesystemsAnswer filesystemsAnswer = (ListFilesystemsAnswer) answer; + return filesystemsAnswer.getFilesystemInfoTOList(); + } + + @Override + public List listBackupFiles(Backup backup, Backup.VolumeInfo volumeInfo, String filesystem, String directory, Boolean isSymlink) { + VolumeVO volumeVO = volumeDao.findByUuidIncludingRemoved(volumeInfo.getUuid()); + InternalBackupDataStoreVO internalBackupDataStoreVO = internalBackupDataStoreDao.findByBackupIdAndVolumeId(backup.getId(), volumeVO.getId()); + InternalBackupJoinVO internalBackupJoinVO = internalBackupJoinDao.findById(backup.getId()); + DataStore dataStore = dataStoreManager.getDataStore(internalBackupJoinVO.getImageStoreId(), DataStoreRole.Image); + DataStoreTO dataStoreTO = dataStore.getTO(); + Set secondaryStorageUrls = getParentSecondaryStorageUrls((BackupVO)backup); + + ListFilesCommand cmd = new ListFilesCommand(volumeVO.getId(), filesystem, directory, internalBackupDataStoreVO.getBackupPath(), (NfsTO) dataStoreTO, isSymlink, + secondaryStorageUrls); + + String errorMessage = String.format("Unable to list files of backup [%s]. Please check the logs.", backup.getUuid()); + EndPoint endPoint = getRandomEndpointToListBackupFilesOrFilesystems(backup, errorMessage, "files"); + + Answer answer = endPoint.sendMessage(cmd); + if (answer == null || !answer.getResult()) { + logger.error("Unable to list files of backup [{}] due to [{}].", backup, answer == null ? "null answer" : answer.getDetails()); + throw new CloudRuntimeException(errorMessage); + } + + ListDataStoreObjectsAnswer listAnswer = (ListDataStoreObjectsAnswer) answer; + + if (!listAnswer.isPathExists()) { + throw new InvalidParameterValueException(String.format("Directory [%s] was not found in backup [%s] of volume [%s].", directory, backup.getUuid(), volumeVO.getUuid())); + } + + List dataStoreObjectResponseList = new ArrayList<>(); + for (int i = 0; i < listAnswer.getCount(); i++) { + DataStoreObjectResponse dataStoreObjectResponse = new DataStoreObjectResponse(listAnswer.getNames().get(i), listAnswer.getIsDirs().get(i), listAnswer.getSizes().get(i), + new Date(listAnswer.getLastModified().get(i))); + + dataStoreObjectResponse.setCanonicalPath(listAnswer.getAbsPaths().get(i)); + dataStoreObjectResponse.setIsSymlink(listAnswer.getIsSymlinks().get(i)); + dataStoreObjectResponse.setVolumeId(volumeVO.getUuid()); + dataStoreObjectResponse.setVolumeName(volumeVO.getName()); + dataStoreObjectResponseList.add(dataStoreObjectResponse); + } + return dataStoreObjectResponseList; + } + + @Override + public String downloadBackupFile(Backup backup, Backup.VolumeInfo volumeInfo, String filesystem, String file) { + BackupOfferingVO offeringVO = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + BackupOfferingVO backupOfferingVO = backupOfferingDao.findByUuidIncludingRemoved(offeringVO.getExternalId()); + BackupOfferingDetailsVO detail = backupOfferingDetailsDao.findDetail(backupOfferingVO.getId(), ApiConstants.ALLOW_EXTRACT_FILE); + if (detail == null || !Boolean.parseBoolean(detail.getValue())) { + throw new CloudRuntimeException(String.format("Unable to download backup files from backup [%s] as its backup offering does not allow it.", backup.getUuid())); + } + + InternalBackupJoinVO internalBackupJoinVO = internalBackupJoinDao.findById(backup.getId()); + ImageStoreEntity dataStore = (ImageStoreEntity) dataStoreManager.getDataStore(internalBackupJoinVO.getImageStoreId(), DataStoreRole.Image); + VolumeVO volumeVO = volumeDao.findByUuidIncludingRemoved(volumeInfo.getUuid()); + InternalBackupDataStoreVO internalBackupDataStoreVO = internalBackupDataStoreDao.findByBackupIdAndVolumeId(backup.getId(), volumeVO.getId()); + DataStoreTO dataStoreTO = dataStore.getTO(); + String backupPath = internalBackupDataStoreVO.getBackupPath(); + + String extractedFile = UUID.nameUUIDFromBytes(file.getBytes()) + ".gz"; + String extractedFileFullPath = backupPath.substring(0, backupPath.lastIndexOf(File.separator)+ 1 ) + extractedFile; + + ImageStoreObjectDownloadVO imageStoreObj = imageStoreObjectDownloadDao.findByStoreIdAndPath(dataStore.getId(), extractedFileFullPath); + if (imageStoreObj != null) { + return imageStoreObj.getDownloadUrl(); + } + Set secondaryStorageUrls = getParentSecondaryStorageUrls((BackupVO)backup); + + ExtractBackupFileCommand cmd = new ExtractBackupFileCommand(volumeVO.getId(), filesystem, file, extractedFileFullPath, backupPath, (NfsTO) dataStoreTO, secondaryStorageUrls); + + EndPoint endPoint = endPointSelector.selectRandom(backup.getZoneId(), Hypervisor.HypervisorType.KVM); + String errorMessage = String.format("Unable to download files of backup [%s]. Please check the logs.", backup.getUuid()); + if (endPoint == null) { + logger.error("Unable to find KVM host to download files of backup [{}]. Check if there is any KVM host that is is UP for the zone.", backup); + throw new CloudRuntimeException(errorMessage); + } + + Answer answer = endPoint.sendMessage(cmd); + if (answer == null || !answer.getResult()) { + logger.error("Unable to download file [{}] of backup [{}] due to [{}].", file, backup, answer == null ? "null answer" : answer.getDetails()); + throw new CloudRuntimeException(errorMessage); + } + + boolean isDir = Boolean.parseBoolean(answer.getDetails()); + BackupFileTO backupFileTO = new BackupFileTO(dataStoreTO, Hypervisor.HypervisorType.KVM, extractedFileFullPath); + BackupFileObject backupFileObject = new BackupFileObject(backupFileTO, dataStore); + try { + String downloadUrl = dataStore.createEntityExtractUrl(extractedFileFullPath, isDir? Storage.ImageFormat.TARGZ : Storage.ImageFormat.GZIP, backupFileObject); + imageStoreObjectDownloadDao.persist(new ImageStoreObjectDownloadVO(dataStore.getId(), extractedFileFullPath, downloadUrl)); + return downloadUrl; + } catch (Exception e) { + logger.warn("Caught exception while trying to create the extract URL of backup file [{}] of backup [{}]. We will cleanup the file and rethrow the exception.", file, + backup.getUuid(), e); + cmd = new ExtractBackupFileCommand(extractedFileFullPath, (NfsTO) dataStoreTO); + answer = endPoint.sendMessage(cmd); + if (!answer.getResult()) { + logger.warn("Unable to cleanup backup file [{}] of backup [{}] due to [{}].", file, backup.getUuid(), answer.getDetails()); + } + throw e; + } + } + @Override public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { return false; @@ -2464,6 +2611,15 @@ protected List getBackupJoinChildren(BackupVO backupVO) { return children; } + private EndPoint getRandomEndpointToListBackupFilesOrFilesystems(Backup backup, String errorMessage, String filesOrFilesystems) { + EndPoint endPoint = endPointSelector.selectRandom(backup.getZoneId(), Hypervisor.HypervisorType.KVM); + if (endPoint == null) { + logger.error("Unable to find KVM host to list {} of backup [{}]. Check if there is any KVM host that is is UP for the zone.", filesOrFilesystems, backup); + throw new CloudRuntimeException(errorMessage); + } + return endPoint; + } + /** * Creates a detail for the given BackupVO if the remaining chain size is one or less and the value of backupChainSize is greater than 0. * */ diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 4281036d9456..3c4fda5f6082 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -7411,5 +7411,19 @@ protected Boolean checkBlockPullProgress(Domain vm, String diskLabel, String vmN return false; } + /** + * Mounts the given secondary storage urls. Returning a reference to the secondary storage "secondaryStorageUrl" and putting the references to the "parentSecondaryStorageUrls" + * into "secondaryStorageUuids" + * */ + public KVMStoragePool mountSecondaryStorages(Set parentSecondaryStorageUrls, String secondaryStorageUrl, KVMStoragePoolManager storagePoolManager, + Set secondaryStorageUuids) { + for (String url : parentSecondaryStorageUrls) { + KVMStoragePool pool = storagePoolManager.getStoragePoolByURI(url); + secondaryStorageUuids.add(pool.getUuid()); + } + KVMStoragePool pool = storagePoolManager.getStoragePoolByURI(secondaryStorageUrl); + secondaryStorageUuids.add(pool.getUuid()); + return pool; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadBackupFileCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadBackupFileCommandWrapper.java new file mode 100644 index 000000000000..d80983a438de --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadBackupFileCommandWrapper.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import org.apache.cloudstack.backup.ExtractBackupFileCommand; +import org.apache.cloudstack.utils.qemu.GuestfishClient; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +@ResourceWrapper(handles = ExtractBackupFileCommand.class) +public class LibvirtDownloadBackupFileCommandWrapper extends CommandWrapper { + @Override + public Answer execute(ExtractBackupFileCommand cmd, LibvirtComputingResource serverResource) { + NfsTO nfs = cmd.getDatastore(); + KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + Set secondaryStorageUuids = new HashSet<>(); + + try { + KVMStoragePool imagePool = serverResource.mountSecondaryStorages(cmd.getSecondaryStorageUrls(), nfs.getUrl(), storagePoolMgr, secondaryStorageUuids); + String fullBackupPath = imagePool.getLocalPathFor(cmd.getBackupPath()); + if (cmd.isCleanup()) { + return cleanupExtractedFile(cmd, imagePool); + } + boolean isDir = false; + try (GuestfishClient guestfishClient = new GuestfishClient(fullBackupPath, cmd.getVolumeId())) { + isDir = guestfishClient.extractFile(cmd.getFilesystem(), cmd.getFilePath(), imagePool.getLocalPathFor(cmd.getDestinationPath()), true); + } + + return new Answer(cmd, true, Boolean.toString(isDir)); + } finally { + for (String uuid : secondaryStorageUuids) { + storagePoolMgr.deleteStoragePool(Storage.StoragePoolType.NetworkFilesystem, uuid); + } + } + } + + private Answer cleanupExtractedFile(ExtractBackupFileCommand cmd, KVMStoragePool imagePool) { + try { + Files.deleteIfExists(Path.of(imagePool.getLocalPathFor(cmd.getDestinationPath()))); + } catch (IOException e) { + return new Answer(cmd, e); + } + return new Answer(cmd); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesCommandWrapper.java new file mode 100644 index 000000000000..925556e28b8b --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesCommandWrapper.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import org.apache.cloudstack.backup.ListFilesCommand; +import org.apache.cloudstack.utils.qemu.GuestfishClient; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@ResourceWrapper(handles = ListFilesCommand.class) +public class LibvirtListFilesCommandWrapper extends CommandWrapper { + @Override + public Answer execute(ListFilesCommand cmd, LibvirtComputingResource serverResource) { + List filesystemInfoTOList = new ArrayList<>(); + NfsTO nfs = cmd.getDatastore(); + KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + Set secondaryStorageUuids = new HashSet<>(); + + try { + KVMStoragePool imagePool = serverResource.mountSecondaryStorages(cmd.getSecondaryStorageUrls(), nfs.getUrl(), storagePoolMgr, secondaryStorageUuids); + + String fullPath = imagePool.getLocalPathFor(cmd.getBackupPath()); + try (GuestfishClient guestfishClient = new GuestfishClient(fullPath, cmd.getVolumeId())) { + return guestfishClient.listFiles(cmd.getFilesystem(), cmd.getDirPath(), cmd.isSymlink()); + } + } finally { + for (String uuid : secondaryStorageUuids) { + storagePoolMgr.deleteStoragePool(Storage.StoragePoolType.NetworkFilesystem, uuid); + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesystemsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesystemsCommandWrapper.java new file mode 100644 index 000000000000..b056cd29d894 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListFilesystemsCommandWrapper.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import org.apache.cloudstack.backup.ListFilesystemsAnswer; +import org.apache.cloudstack.backup.ListFilesystemsCommand; +import org.apache.cloudstack.utils.qemu.GuestfishClient; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@ResourceWrapper(handles = ListFilesystemsCommand.class) +public class LibvirtListFilesystemsCommandWrapper extends CommandWrapper { + @Override + public Answer execute(ListFilesystemsCommand cmd, LibvirtComputingResource serverResource) { + List filesystemInfoTOList = new ArrayList<>(); + NfsTO nfs = cmd.getDatastore(); + KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + Set secondaryStorageUuids = new HashSet<>(); + + try { + KVMStoragePool imagePool = serverResource.mountSecondaryStorages(cmd.getSecondaryStorageUrls(), nfs.getUrl(), storagePoolMgr, secondaryStorageUuids); + for (Long volumeId : cmd.getVolumeIdAndBackupPath().keySet()) { + String fullPath = imagePool.getLocalPathFor(cmd.getVolumeIdAndBackupPath().get(volumeId)); + try (GuestfishClient guestfishClient = new GuestfishClient(fullPath, volumeId)) { + filesystemInfoTOList.addAll(guestfishClient.listFilesystems()); + } + } + + return new ListFilesystemsAnswer(cmd, filesystemInfoTOList); + } finally { + for (String uuid : secondaryStorageUuids) { + storagePoolMgr.deleteStoragePool(Storage.StoragePoolType.NetworkFilesystem, uuid); + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapper.java index a9010b46c1d4..5fb6e067c7cb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapper.java @@ -54,8 +54,8 @@ public Answer execute(RestoreKbossBackupCommand cmd, LibvirtComputingResource re Set secondaryStorageUuids = new HashSet<>(); try { - KVMStoragePool secondaryStorage = mountSecondaryStorages(secondaryStorageUrls, backupToAndVolumeObjectPairs.stream().findFirst().get().first().getDataStore().getUrl(), - storagePoolManager, secondaryStorageUuids); + KVMStoragePool secondaryStorage = resource.mountSecondaryStorages(secondaryStorageUrls, + backupToAndVolumeObjectPairs.stream().findFirst().get().first().getDataStore().getUrl(), storagePoolManager, secondaryStorageUuids); restoreVolumes(backupToAndVolumeObjectPairs, secondaryStorage, storagePoolManager, cmd.isQuickRestore(), cmd.getWait() * 1000); @@ -110,14 +110,4 @@ protected void deleteDeltas(Set deltasToRemove, KVMStoragePoolMan Files.deleteIfExists(Path.of(fullDeltaPath)); } } - - protected KVMStoragePool mountSecondaryStorages(Set parentSecondaryStorageUrls, String secondaryStorageUrl, KVMStoragePoolManager storagePoolManager, Set secondaryStorageUuids) { - for (String url : parentSecondaryStorageUrls) { - KVMStoragePool pool = storagePoolManager.getStoragePoolByURI(url); - secondaryStorageUuids.add(pool.getUuid()); - } - KVMStoragePool pool = storagePoolManager.getStoragePoolByURI(secondaryStorageUrl); - secondaryStorageUuids.add(pool.getUuid()); - return pool; - } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/GuestfishClient.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/GuestfishClient.java new file mode 100644 index 000000000000..d9f798182d6c --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/GuestfishClient.java @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.qemu; + +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class was built to use guestfish commands on qcow2 volumes. + * It should always be instanced with try-with-resources + * Otherwise, be sure to call close(). + * */ + +public class GuestfishClient implements AutoCloseable { + private Logger logger = LogManager.getLogger(getClass()); + + private static final int MODE_MASK = 0_170_000; + private static final int SYMLINK = 0_120_000; + private static final int DIRECTORY = 0_040_000; + private static final int FILE = 0_100_000; + + private static final String GUESTFISH = "guestfish"; + private static final String GUESTFISH_PID_STRING = "GUESTFISH_PID="; + private static final String MODE = "st_mode"; + private static final String SIZE = "st_size"; + private static final String ATIME_SEC = "st_atime_sec"; + + // Guestfish commands + protected static final String MOUNT_RO = "mount-ro"; + protected static final String UMOUNT_ALL = "umount-all"; + protected static final String TAR_OUT = "tar-out"; + protected static final String COMPRESS_OUT = "compress-out"; + protected static final String LIST_FILESYSTEMS = "list-filesystems"; + protected static final String LSTATNS = "lstatns"; + protected static final String LSTATNSLIST = "lstatnslist"; + protected static final String IS_SYMLINK = "is-symlink"; + protected static final String READLINK = "readlink"; + protected static final String BLOCKDEV_GETSIZE_64 = "blockdev-getsize64"; + protected static final String LS = "ls"; + protected static final String EXISTS = "exists"; + + private static final int MILI = 1000; + + private final String guestfishPid; + private final long volumeId; + + /** + * No-arg constructor for test purposes + */ + protected GuestfishClient() { + guestfishPid = "0"; + volumeId = 0; + } + + public GuestfishClient(String qcow2Path, long volumeId) { + this.volumeId = volumeId; + + Script script = new Script(GUESTFISH); + script.add("--listen"); + script.add("--ro"); + + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String scriptResult = script.execute(parser); + if (scriptResult != null) { + throw new CloudRuntimeException("Could not start guestfish. Is it installed?"); + } + + String output = parser.getLine(); + + this.guestfishPid = parseGuestfishPid(output); + + if (guestfishPid == null) { + throw new CloudRuntimeException("Could not parse guestfish PID."); + } + + runGuestfishRemoteCommand("add-drive-ro", qcow2Path); + runGuestfishRemoteCommand("run"); + } + + /** + * List filesystems. Will not return swap or unknown filesystems. + */ + public List listFilesystems() { + String output = runGuestfishRemoteCommand(LIST_FILESYSTEMS); + List result = new ArrayList<>(); + + logger.debug("Got the following output from list-filesystems: [{}]", output); + output.lines().forEach( line ->{ + line = line.trim(); + if (line.isEmpty()) { + return; + } + + String[] parts = line.split(":"); + if (parts.length != 2) { + return; + } + + String device = parts[0].trim(); + String type = parts[1].trim(); + if (type.equalsIgnoreCase("swap") || type.equalsIgnoreCase("unknown")) { + logger.debug("Ignoring filesystem [{}] with [{}] type.", device, type); + return; + } + + String size = runGuestfishRemoteCommand(BLOCKDEV_GETSIZE_64, device); + + result.add(new FilesystemInfoTO(device, type, parseLong(size), volumeId)); + }); + + return result; + } + + /** + * List files in the given directory. If the directory is a symlink, will try to resolve it, if we are unable to resolve it in one hop, we will return an error. + * */ + public ListDataStoreObjectsAnswer listFiles(String filesystem, String directory, Boolean isSymlink) { + try { + mount(filesystem); + + if (isSymlink == null) { + String isSymlinkString = runGuestfishRemoteCommand(IS_SYMLINK, directory); + isSymlink = Boolean.parseBoolean(isSymlinkString); + } + + if (isSymlink) { + logger.debug("Directory [{}] is a symlink, will try resolve it.", directory); + directory = (directory.charAt(directory.length() - 1) == '/' ? directory.substring(0, directory.length() - 1) : directory); + String canonicalPath = getCanonicalPath(directory); + if (canonicalPath == null) { + return new ListDataStoreObjectsAnswer(); + } + + String isSymlinkString = runGuestfishRemoteCommand(IS_SYMLINK, canonicalPath); + isSymlink = Boolean.parseBoolean(isSymlinkString); + if (isSymlink) { + logger.warn("Directory [{}] is a symlink chain. Unable to list its files. Please try to list the real directory.", directory); + return new ListDataStoreObjectsAnswer(); + } + logger.debug("Directory [{}] was resolved to [{}].", directory, canonicalPath); + directory = canonicalPath; + } + + String lsOutput = runGuestfishRemoteCommand(LS, directory); + + List fileList = lsOutput.lines().filter(StringUtils::isNotBlank).collect(Collectors.toList()); + + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List canonicalPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List isSymlinks = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + + if (fileList.isEmpty()) { + return new ListDataStoreObjectsAnswer(true, 0, names, paths, canonicalPaths, isDirs, sizes, modifiedList); + } + + String details = getDetails(directory, fileList); + logger.trace("Got the following details for these files [{}]: [{}]", fileList, details); + + addFilesToLists(directory, details, fileList, names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + + return new ListDataStoreObjectsAnswer(true, names.size(), names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + } finally { + runGuestfishRemoteCommand(UMOUNT_ALL); + } + } + + /** + * Extracts a file from the given filesystem and path to the destination. If followSymlink is true will try to resolve it, if we cannot do it in one hop we throw an error. + * */ + public boolean extractFile(String filesystem, String filePath, String destination, boolean followSymlink) { + mount(filesystem); + + String details = runGuestfishRemoteCommand(LSTATNS, filePath); + Map stat = parseKeyValueOutput(details); + long mode = parseLong(stat.get(MODE)); + long fileType = mode & MODE_MASK; + boolean isSymlink = fileType == SYMLINK; + + if (isSymlink && followSymlink) { + logger.debug("File is a symlink, will try to resolve it in order to download the actual file."); + String canonicalPath = getCanonicalPath(filePath); + if (canonicalPath == null) { + runGuestfishRemoteCommand(UMOUNT_ALL); + throw new CloudRuntimeException(String.format("File [%s] is a symlink to a non-existent file. Unable to download.", fileType)); + } + + return extractFile(filesystem, canonicalPath, destination, false); + } else if (isSymlink) { + runGuestfishRemoteCommand(UMOUNT_ALL); + throw new CloudRuntimeException(String.format("Unable to extract file at [%s]. It seems like a symlink chain. Try to download the actual file at the end of the chain.", + filePath)); + } + + boolean isDirectory = fileType == DIRECTORY; + boolean isFile = fileType == FILE; + try { + if (isDirectory) { + logger.debug("Extracting directory at [{}] to [{}].", filePath, destination); + runGuestfishRemoteCommand(TAR_OUT, filePath, destination, "compress:gzip"); + } else if (isFile) { + logger.debug("Extracting file at [{}] to [{}].", filePath, destination); + runGuestfishRemoteCommand(COMPRESS_OUT, "gzip", filePath, destination); + } else { + throw new CloudRuntimeException(String.format("Unable to extract file at [%s]. It is neither a file nor a directory.", filePath)); + } + } catch (Exception e) { + logger.error("Caught exception while extracting file, will try to delete any leftovers and rethrow the exception.", e); + try { + Files.deleteIfExists(Path.of(destination)); + } catch (IOException ignored) { + } + throw e; + } finally { + runGuestfishRemoteCommand(UMOUNT_ALL); + } + return isDirectory; + } + + protected String getDetails(String directory, List fileList) { + String fileListString = formatFileListForLstatnslist(fileList); + + return runGuestfishRemoteCommand(LSTATNSLIST, directory, fileListString); + } + + + protected String formatFileListForLstatnslist(List fileList) { + StringBuilder fileListStringBuilder = new StringBuilder(); + for (String string : fileList) { + fileListStringBuilder.append("'"); + fileListStringBuilder.append(string); + fileListStringBuilder.append("'"); + fileListStringBuilder.append(" "); + } + fileListStringBuilder.deleteCharAt(fileListStringBuilder.length() - 1); + return fileListStringBuilder.toString(); + } + + /** + * Given the directory, file list and details of those files, will populate the names, paths, canonicalPaths, isDirs, isSymlinks, sizes and modifiedList lists. + * If a file is a symlink, will try to resolve it in a single hop, if unable to, we will not add it to the list; furthermore, if the symlink points to an inexistent file, we + * will not add it to the list either. + */ + protected void addFilesToLists(String directory, String details, List fileList, List names, List paths, List canonicalPaths, + List isDirs, List isSymlinks, List sizes, List modifiedList) { + String[] tokens = details.split("="); + for (int i = 1; i < tokens.length; i++) { + Map stat = parseKeyValueOutput(tokens[i]); + + long mode = parseLong(stat.get(MODE)); + long fileType = mode & MODE_MASK; + boolean isSymlink = fileType == SYMLINK; + + String path = (directory.charAt(directory.length() - 1) == '/' ? directory : directory + '/') + fileList.get(i - 1); + String canonicalPath = path; + if (isSymlink) { + logger.debug("File [{}] is a symlink, will try to resolve it and add it to the list of files of the backup.", path); + canonicalPath = getCanonicalPath(path); + if (canonicalPath == null) { + continue; + } + + String fileDetails = runGuestfishRemoteCommand(LSTATNS, canonicalPath); + stat = parseKeyValueOutput(fileDetails); + mode = parseLong(stat.get(MODE)); + + fileType = mode & MODE_MASK; + if (fileType == SYMLINK) { + logger.warn("File [{}] is a symlink chain, will not return it on the list of files of the backup.", path); + continue; + } else { + logger.debug("Symlink at [{}] was resolved to [{}] in the list of files of the backup. Setting it as its canonical path", path, canonicalPath); + } + } + names.add(fileList.get(i - 1)); + paths.add(path); + canonicalPaths.add(canonicalPath); + + boolean isDirectory = fileType == DIRECTORY; + isDirs.add(isDirectory); + isSymlinks.add(isSymlink); + + long size = parseLong(stat.get(SIZE)); + sizes.add(size); + long mtime = parseLong(stat.get(ATIME_SEC)); + modifiedList.add(mtime * MILI); + } + } + + /** + * @param filePath path of the symlink to try and get the canonical path from. If it points to an unexisting file, we return null. This method expects the filePath to be a + * symlink and will throw an error if it is not. + * @return The canonical path, if the file exists; null otherwise. + */ + protected String getCanonicalPath(String filePath) { + String canonicalPath = runGuestfishRemoteCommand(READLINK, filePath); + canonicalPath = canonicalPath.replace("/sysroot", ""); + if (canonicalPath.charAt(0) != '/') { + String basepath = filePath.substring(0, filePath.lastIndexOf('/')); + canonicalPath = Paths.get(basepath + "/" + canonicalPath).normalize().toString(); + } + + if (!Boolean.parseBoolean(runGuestfishRemoteCommand(EXISTS, canonicalPath))) { + logger.warn("Symlink [{}] points to a file that does not exist.", filePath); + return null; + } + + return canonicalPath; + } + + protected void mount(String filesystem) { + runGuestfishRemoteCommand(UMOUNT_ALL); + runGuestfishRemoteCommand(MOUNT_RO, filesystem, "/"); + } + + /** + * Execute remote guestfish command + */ + protected String runGuestfishRemoteCommand(String... commandParts) { + Script script = new Script(GUESTFISH); + script.add("--remote=" + guestfishPid); + script.add("--"); + script.add(commandParts); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String scriptResult = script.execute(parser); + if (scriptResult != null) { + throw new CloudRuntimeException(String.format("Got unexpected output when trying to run guestfish command [%s].", script)); + } + + String result = parser.getLines(); + logger.trace("Result from guestfish command [{}] is [{}].", script.toString(), result); + return result.trim(); + } + + protected String parseGuestfishPid(String output) { + for (String token : output.split(";")) { + token = token.trim(); + + if (token.startsWith(GUESTFISH_PID_STRING)) { + return token.substring(GUESTFISH_PID_STRING.length()); + } + } + return null; + } + + protected Map parseKeyValueOutput(String output) { + Map map = new HashMap<>(); + + output.lines().forEach(line -> { + line = line.trim(); + if (line.isEmpty()) { + return; + } + + int index = line.indexOf(':'); + if (index < 0) { + return; + } + + String key = line.substring(0, index).trim(); + String value = line.substring(index + 1).trim(); + map.put(key, value); + }); + + return map; + } + + protected long parseLong(String value) { + if (StringUtils.isEmpty(value)) { + return 0; + } + + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + logger.warn("Unable to parse long [{}]. Returning 0.", value); + return 0; + } + } + + + @Override + public void close() { + runGuestfishRemoteCommand("exit"); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapperTest.java index 75668ad15f3b..3b6dba3ec9cc 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreKbossBackupCommandWrapperTest.java @@ -109,7 +109,7 @@ public class LibvirtRestoreKbossBackupCommandWrapperTest { public void executeTestException() throws LibvirtException, QemuImgException { doReturn(primaryDataStoreToMock).when(backupDeltaTOMock).getDataStore(); doReturn(Set.of(new Pair<>(backupDeltaTOMock, volumeObjectToMock1))).when(cmdMock).getBackupAndVolumePairs(); - doReturn(null).when(libvirtRestoreKbossBackupCommandWrapperSpy).mountSecondaryStorages(any(), any(), any(), any()); + doReturn(null).when(libvirtComputingResourceMock).mountSecondaryStorages(any(), any(), any(), any()); doThrow(new QemuImgException("asd")).when(libvirtRestoreKbossBackupCommandWrapperSpy).restoreVolumes(any(), any(), any(), anyBoolean(), anyInt()); RestoreKbossBackupAnswer answer = (RestoreKbossBackupAnswer)libvirtRestoreKbossBackupCommandWrapperSpy.execute(cmdMock, libvirtComputingResourceMock); @@ -122,8 +122,6 @@ public void executeTestLibvirtNotQuickRestore() throws LibvirtException, QemuImg doReturn(primaryDataStoreToMock).when(backupDeltaTOMock).getDataStore(); doReturn(Set.of(new Pair<>(backupDeltaTOMock, volumeObjectToMock1))).when(cmdMock).getBackupAndVolumePairs(); doReturn(kvmStoragePoolManagerMock).when(libvirtComputingResourceMock).getStoragePoolMgr(); - doReturn(kvmStoragePool1).when(kvmStoragePoolManagerMock).getStoragePoolByURI(any()); - doReturn("uuid").when(kvmStoragePool1).getUuid(); doNothing().when(libvirtRestoreKbossBackupCommandWrapperSpy).restoreVolumes(any(), any(), any(), anyBoolean(), anyInt()); doNothing().when(libvirtRestoreKbossBackupCommandWrapperSpy).deleteDeltas(any(), any()); @@ -131,7 +129,7 @@ public void executeTestLibvirtNotQuickRestore() throws LibvirtException, QemuImg RestoreKbossBackupAnswer answer = (RestoreKbossBackupAnswer)libvirtRestoreKbossBackupCommandWrapperSpy.execute(cmdMock, libvirtComputingResourceMock); assertTrue(answer.getResult()); - verify(kvmStoragePoolManagerMock).deleteStoragePool(Storage.StoragePoolType.NetworkFilesystem, "uuid"); + verify(kvmStoragePoolManagerMock, never()).deleteStoragePool(Storage.StoragePoolType.NetworkFilesystem, "uuid"); } @@ -140,8 +138,6 @@ public void executeTestLibvirtQuickRestore() throws LibvirtException, QemuImgExc doReturn(primaryDataStoreToMock).when(backupDeltaTOMock).getDataStore(); doReturn(Set.of(new Pair<>(backupDeltaTOMock, volumeObjectToMock1))).when(cmdMock).getBackupAndVolumePairs(); doReturn(kvmStoragePoolManagerMock).when(libvirtComputingResourceMock).getStoragePoolMgr(); - doReturn(kvmStoragePool1).when(kvmStoragePoolManagerMock).getStoragePoolByURI(any()); - doReturn("uuid").when(kvmStoragePool1).getUuid(); doNothing().when(libvirtRestoreKbossBackupCommandWrapperSpy).restoreVolumes(any(), any(), any(), anyBoolean(), anyInt()); doNothing().when(libvirtRestoreKbossBackupCommandWrapperSpy).deleteDeltas(any(), any()); diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/GuestfishClientTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/GuestfishClientTest.java new file mode 100644 index 000000000000..c4c346077e0b --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/GuestfishClientTest.java @@ -0,0 +1,467 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.qemu; + +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class GuestfishClientTest { + + @Spy + GuestfishClient guestfishClientSpy; + + private static final String statsDetailsDir = "st_dev: 2049\nst_ino: 2\nst_mode: 16877\nst_nlink: 22\nst_uid: 0\nst_gid: 0\nst_rdev: 0\nst_size: 4096\n" + + "st_blksize: 4096\nst_blocks: 8\nst_atime_sec: 1779800014\nst_atime_nsec: 952000000\nst_mtime_sec: 1779802005\nst_mtime_nsec: 508000000\nst_ctime_sec: 1779802005\n" + + "st_ctime_nsec: 508000000\nst_spare1: 0\nst_spare2: 0\nst_spare3: 0\nst_spare4: 0\nst_spare5: 0\nst_spare6: 0"; + + private static final String statsDetailsFile = "st_dev: 2049\nst_ino: 2\nst_mode: 32768\nst_nlink: 22\nst_uid: 0\nst_gid: 0\nst_rdev: 0\nst_size: 4096\n" + + "st_blksize: 4096\nst_blocks: 8\nst_atime_sec: 1779800014\nst_atime_nsec: 952000000\nst_mtime_sec: 1779802005\nst_mtime_nsec: 508000000\nst_ctime_sec: 1779802005\n" + + "st_ctime_nsec: 508000000\nst_spare1: 0\nst_spare2: 0\nst_spare3: 0\nst_spare4: 0\nst_spare5: 0\nst_spare6: 0"; + + private static final String statsDetailsSymlink = "st_dev: 2049\nst_ino: 2\nst_mode: 40960\nst_nlink: 22\nst_uid: 0\nst_gid: 0\nst_rdev: 0\nst_size: 4096\n" + + "st_blksize: 4096\nst_blocks: 8\nst_atime_sec: 1779800014\nst_atime_nsec: 952000000\nst_mtime_sec: 1779802005\nst_mtime_nsec: 508000000\nst_ctime_sec: 1779802005\n" + + "st_ctime_nsec: 508000000\nst_spare1: 0\nst_spare2: 0\nst_spare3: 0\nst_spare4: 0\nst_spare5: 0\nst_spare6: 0"; + + private static final String statsDetailsUnknownType = "st_dev: 2049\nst_ino: 2\nst_mode: 60\nst_nlink: 22\nst_uid: 0\nst_gid: 0\nst_rdev: 0\nst_size: 4096\n" + + "st_blksize: 4096\nst_blocks: 8\nst_atime_sec: 1779800014\nst_atime_nsec: 952000000\nst_mtime_sec: 1779802005\nst_mtime_nsec: 508000000\nst_ctime_sec: 1779802005\n" + + "st_ctime_nsec: 508000000\nst_spare1: 0\nst_spare2: 0\nst_spare3: 0\nst_spare4: 0\nst_spare5: 0\nst_spare6: 0"; + + @Test + public void parseLongTestNull() { + long result = guestfishClientSpy.parseLong(null); + assertEquals(0, result); + } + + @Test + public void parseLongTestEmpty() { + long result = guestfishClientSpy.parseLong(""); + assertEquals(0, result); + } + + @Test + public void parseLongTestNaN() { + long result = guestfishClientSpy.parseLong("acs"); + assertEquals(0, result); + } + + @Test + public void parseLongTestActualNumber() { + long result = guestfishClientSpy.parseLong("123"); + assertEquals(123, result); + } + + @Test + public void parseKeyValueOutputTestEmptyString() { + String stringToParse = ""; + Map map = guestfishClientSpy.parseKeyValueOutput(stringToParse); + assertEquals(Map.of(), map); + } + + @Test + public void parseKeyValueOutputTestUnexpectedString() { + String stringToParse = "a=n\nc=d\n"; + Map map = guestfishClientSpy.parseKeyValueOutput(stringToParse); + assertEquals(Map.of(), map); + } + + @Test + public void parseKeyValueOutputTestWithExpectedInput() { + Map map = guestfishClientSpy.parseKeyValueOutput(statsDetailsDir); + assertEquals(2049, Long.parseLong(map.get("st_dev"))); + assertEquals(1779800014, Long.parseLong(map.get("st_atime_sec"))); + assertEquals(4096, Long.parseLong(map.get("st_size"))); + assertEquals(0, Long.parseLong(map.get("st_spare1"))); + assertEquals(16877, Long.parseLong(map.get("st_mode"))); + } + + @Test + public void getCanonicalPathTestFileDoesNotExist() { + String input = "/bin"; + doReturn("/log").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.READLINK, input); + doReturn("false").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.EXISTS, "/log"); + + String result = guestfishClientSpy.getCanonicalPath(input); + + assertNull(result); + } + + @Test + public void getCanonicalPathTestNtfsStyleCanonicalPath() { + String input = "/bin"; + doReturn("/sysroot/log").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.READLINK, input); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.EXISTS, "/log"); + + String result = guestfishClientSpy.getCanonicalPath(input); + + assertEquals("/log", result); + } + + @Test + public void getCanonicalPathTestRelativePathTwoDots() { + String input = "/bin/test"; + doReturn("../log").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.READLINK, input); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.EXISTS, "/log"); + + String result = guestfishClientSpy.getCanonicalPath(input); + + assertEquals("/log", result); + } + + @Test + public void getCanonicalPathTestRelativePathSingleDot() { + String input = "/bin/test"; + doReturn("./log").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.READLINK, input); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.EXISTS, "/bin/log");; + + String result = guestfishClientSpy.getCanonicalPath(input); + + assertEquals("/bin/log", result); + } + + @Test + public void getCanonicalPathTestRelativePath() { + String input = "/bin/test"; + doReturn("log").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.READLINK, input); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.EXISTS, "/bin/log"); + + String result = guestfishClientSpy.getCanonicalPath(input); + + assertEquals("/bin/log", result); + } + + @Test + public void addFilesToListsTestNormalFile() { + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List canonicalPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List isSymlinks = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + String directory = "/"; + String details = "[0] = {" + statsDetailsDir + "}"; + List fileList = List.of("boot"); + + guestfishClientSpy.addFilesToLists(directory, details, fileList, names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + + assertEquals("boot", names.get(0)); + assertEquals("/boot", paths.get(0)); + assertEquals("/boot", canonicalPaths.get(0)); + assertEquals(true, isDirs.get(0)); + assertEquals(false, isSymlinks.get(0)); + assertEquals((Long)4096L, sizes.get(0)); + assertEquals((Long)1779800014000L, modifiedList.get(0)); + } + + @Test + public void addFilesToListsTestSymlinkFile() { + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List canonicalPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List isSymlinks = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + String directory = "/"; + String details = "[0] = {" + statsDetailsSymlink + "}"; + List fileList = List.of("boot"); + doReturn("/tst/path").when(guestfishClientSpy).getCanonicalPath("/boot"); + doReturn(statsDetailsDir).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, "/tst/path"); + + guestfishClientSpy.addFilesToLists(directory, details, fileList, names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + + assertEquals("boot", names.get(0)); + assertEquals("/boot", paths.get(0)); + assertEquals("/tst/path", canonicalPaths.get(0)); + assertEquals(true, isDirs.get(0)); + assertEquals(true, isSymlinks.get(0)); + assertEquals((Long)4096L, sizes.get(0)); + assertEquals((Long)1779800014000L, modifiedList.get(0)); + } + + @Test + public void addFilesToListsTestSymlinkFileToAnotherSymlink() { + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List canonicalPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List isSymlinks = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + String directory = "/"; + String details = "[0] = {" + statsDetailsSymlink + "}"; + List fileList = List.of("boot"); + doReturn("/tst/path").when(guestfishClientSpy).getCanonicalPath("/boot"); + doReturn(statsDetailsSymlink).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, "/tst/path"); + + guestfishClientSpy.addFilesToLists(directory, details, fileList, names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + + assertEquals(List.of(), names); + assertEquals(List.of(), paths); + assertEquals(List.of(), canonicalPaths); + assertEquals(List.of(), isDirs); + assertEquals(List.of(), isSymlinks); + assertEquals(List.of(), sizes); + assertEquals(List.of(), modifiedList); + } + + @Test + public void formatFileListForLstatnslistTest() { + List fileList = List.of("libvirt", "super lib"); + + String result = guestfishClientSpy.formatFileListForLstatnslist(fileList); + + assertEquals("'libvirt' 'super lib'", result); + } + + @Test + public void extractFileTestExtractDir() { + String filesystem = "/dev/sda"; + String filePath = "/usr/share/batata"; + String destination = "/mnt/sec/kpodkpo.gz"; + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn(statsDetailsDir).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.TAR_OUT, filePath, destination, "compress:gzip"); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + boolean result = guestfishClientSpy.extractFile(filesystem, filePath, destination, false); + + assertTrue(result); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.TAR_OUT, filePath, destination, "compress:gzip"); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + } + + @Test + public void extractFileTestExtractSymlinkToFile() { + String filesystem = "/dev/sda"; + String filePath = "/usr/share/batata"; + String canonicalFilePath = "/etc/frita"; + String destination = "/mnt/sec/kpodkpo.gz"; + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn(statsDetailsSymlink).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + doReturn(canonicalFilePath).when(guestfishClientSpy).getCanonicalPath(filePath); + doReturn(statsDetailsFile).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, canonicalFilePath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.COMPRESS_OUT, "gzip", canonicalFilePath, destination); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + boolean result = guestfishClientSpy.extractFile(filesystem, filePath, destination, true); + + assertFalse(result); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + verify(guestfishClientSpy).getCanonicalPath(filePath); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, canonicalFilePath); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.COMPRESS_OUT, "gzip", canonicalFilePath, destination); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + } + + @Test (expected = CloudRuntimeException.class) + public void extractFileTestExtractUnknownFileType() { + String filesystem = "/dev/sda"; + String filePath = "/usr/share/batata"; + String destination = "/mnt/sec/kpodkpo.gz"; + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn(statsDetailsUnknownType).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + guestfishClientSpy.extractFile(filesystem, filePath, destination, true); + + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.COMPRESS_OUT), any(), any(), any()); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.TAR_OUT), any(), any(), any()); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + } + + @Test (expected = CloudRuntimeException.class) + public void extractFileTestExtractSymlinkToNonExistentFile() { + String filesystem = "/dev/sda"; + String filePath = "/usr/share/batata"; + String destination = "/mnt/sec/kpodkpo.gz"; + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn(statsDetailsSymlink).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + doReturn(null).when(guestfishClientSpy).getCanonicalPath(filePath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + guestfishClientSpy.extractFile(filesystem, filePath, destination, true); + + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + verify(guestfishClientSpy).getCanonicalPath(filePath); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.COMPRESS_OUT), any(), any(), any()); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.TAR_OUT), any(), any(), any()); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + } + + @Test (expected = CloudRuntimeException.class) + public void extractFileTestExtractSymlinkToSymlink() { + String filesystem = "/dev/sda"; + String filePath = "/usr/share/batata"; + String canonicalFilePath = "/etc/frita"; + String destination = "/mnt/sec/kpodkpo.gz"; + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn(statsDetailsSymlink).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + doReturn(canonicalFilePath).when(guestfishClientSpy).getCanonicalPath(filePath); + doReturn(statsDetailsSymlink).when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, canonicalFilePath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + guestfishClientSpy.extractFile(filesystem, filePath, destination, true); + + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, filePath); + verify(guestfishClientSpy).getCanonicalPath(filePath); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LSTATNS, canonicalFilePath); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.COMPRESS_OUT), any(), any(), any()); + verify(guestfishClientSpy, never()).runGuestfishRemoteCommand(eq(GuestfishClient.TAR_OUT), any(), any(), any()); + verify(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + } + + @Test + public void listFilesTestSymlinkToNonExistentFile() { + String filesystem = "/dev/vdb"; + String directory = "/User Data/"; + Boolean isSymlink = null; + + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.IS_SYMLINK, directory); + doReturn(null).when(guestfishClientSpy).getCanonicalPath(directory.substring(0, directory.length() - 1)); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + ListDataStoreObjectsAnswer result = guestfishClientSpy.listFiles(filesystem, directory, isSymlink); + + assertEquals(0, result.getCount()); + assertFalse(result.isPathExists()); + } + + @Test + public void listFilesTestSymlinkToSymlink() { + String filesystem = "/dev/vdb"; + String directory = "/User Data/"; + String canonicalPath = "/Users/Default/AppData"; + Boolean isSymlink = null; + + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.IS_SYMLINK, directory); + doReturn(canonicalPath).when(guestfishClientSpy).getCanonicalPath(directory.substring(0, directory.length() - 1)); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.IS_SYMLINK, canonicalPath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + ListDataStoreObjectsAnswer result = guestfishClientSpy.listFiles(filesystem, directory, isSymlink); + + assertEquals(0, result.getCount()); + assertFalse(result.isPathExists()); + } + + @Test + public void listFilesTestSymlinkToEmptyDir() { + String filesystem = "/dev/vdb"; + String directory = "/User Data/"; + String canonicalPath = "/Users/Default/AppData"; + Boolean isSymlink = null; + + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn("true").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.IS_SYMLINK, directory); + doReturn(canonicalPath).when(guestfishClientSpy).getCanonicalPath(directory.substring(0, directory.length() - 1)); + doReturn("false").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.IS_SYMLINK, canonicalPath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LS, canonicalPath); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + ListDataStoreObjectsAnswer result = guestfishClientSpy.listFiles(filesystem, directory, isSymlink); + + assertEquals(0, result.getCount()); + assertTrue(result.isPathExists()); + } + + @Test + public void listFilesTestDir() { + String filesystem = "/dev/vdb"; + String directory = "/User Data/"; + Boolean isSymlink = false; + + doNothing().when(guestfishClientSpy).mount(filesystem); + doReturn("a").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LS, directory); + doReturn("[0] = " + statsDetailsFile).when(guestfishClientSpy).getDetails(eq(directory), eq(List.of("a"))); + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.UMOUNT_ALL); + + ListDataStoreObjectsAnswer result = guestfishClientSpy.listFiles(filesystem, directory, isSymlink); + + assertEquals(1, result.getCount()); + assertEquals("a", result.getNames().get(0)); + assertTrue(result.isPathExists()); + } + + @Test + public void listFilesystemsTestNoFilesystem() { + doReturn("").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LIST_FILESYSTEMS); + + List result = guestfishClientSpy.listFilesystems(); + + assertTrue(result.isEmpty()); + } + + @Test + public void listFilesystemsTestSingleFilesystem() { + doReturn("/dev/sda2: ext4").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LIST_FILESYSTEMS); + doReturn("123").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.BLOCKDEV_GETSIZE_64, "/dev/sda2"); + + List result = guestfishClientSpy.listFilesystems(); + + FilesystemInfoTO filesystemInfoTO = result.get(0); + assertEquals("/dev/sda2", filesystemInfoTO.getName()); + assertEquals(123, filesystemInfoTO.getSize()); + assertEquals("ext4", filesystemInfoTO.getFilesystem()); + } + + @Test + public void listFilesystemsTestMultipleFilesystemWithUnknown() { + doReturn("/dev/sda1: unknown\n" + "/dev/sda2: ext4\n" + "/dev/ubuntu-vg/ubuntu-lv: ext3").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.LIST_FILESYSTEMS); + doReturn("123").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.BLOCKDEV_GETSIZE_64, "/dev/sda2"); + doReturn("444").when(guestfishClientSpy).runGuestfishRemoteCommand(GuestfishClient.BLOCKDEV_GETSIZE_64, "/dev/ubuntu-vg/ubuntu-lv"); + + List result = guestfishClientSpy.listFilesystems(); + + FilesystemInfoTO filesystemInfoTO = result.remove(0); + assertEquals("/dev/sda2", filesystemInfoTO.getName()); + assertEquals(123, filesystemInfoTO.getSize()); + assertEquals("ext4", filesystemInfoTO.getFilesystem()); + + FilesystemInfoTO filesystemInfoTO2 = result.remove(0); + assertEquals("/dev/ubuntu-vg/ubuntu-lv", filesystemInfoTO2.getName()); + assertEquals(444, filesystemInfoTO2.getSize()); + assertEquals("ext3", filesystemInfoTO2.getFilesystem()); + + assertTrue(result.isEmpty()); + } +} diff --git a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java index 2d17fd029a14..9916291311cc 100644 --- a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java @@ -68,6 +68,7 @@ public class ParamProcessWorker implements DispatchWorker { private static final String newInputFormatString = "yyyy-MM-dd HH:mm:ss"; public static final DateFormat inputFormat = new SimpleDateFormat(inputFormatString); public static final DateFormat newInputFormat = new SimpleDateFormat(newInputFormatString); + private static final String REGEX = "[$|&*`\\@!%'\"^;<>!()]"; @Inject protected AccountManager _accountMgr; @@ -126,6 +127,14 @@ private void validateNameForRFCCompliance(final Object param, final String argNa } } + private void validateLimitedSpecialCharacters(final Object param, final String argName) { + String value = String.valueOf(param).trim(); + + if (value.matches(REGEX)) { + throwInvalidParameterValueException(argName, String.format("This parameter cannot contain any of these characters: %s.", REGEX)); + } + } + protected void throwInvalidParameterValueException(String argName) { throwInvalidParameterValueException(argName, null); } @@ -174,6 +183,13 @@ private void validateField(final Object paramObj, final Parameter annotation) th validateNameForRFCCompliance(paramObj, argName); break; } + case LimitedSpecialCharacters: + switch (annotation.type()) { + case STRING: + validateLimitedSpecialCharacters(paramObj, argName); + break; + } + break; } } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 58e435b6406d..56792a3f898d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -35,7 +35,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.cloud.agent.api.to.FilesystemInfoTO; import com.cloud.host.Host; +import com.cloud.storage.Upload; import com.cloud.storage.VolumeApiService; import com.cloud.utils.exception.BackupProviderException; import com.cloud.utils.fsm.NoTransitionException; @@ -63,8 +65,11 @@ import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.DownloadBackupFileCmd; import org.apache.cloudstack.api.command.user.backup.DownloadValidationScreenshotCmd; import org.apache.cloudstack.api.command.user.backup.FinishBackupChainCmd; +import org.apache.cloudstack.api.command.user.backup.ListBackupFilesCmd; +import org.apache.cloudstack.api.command.user.backup.ListBackupFilesystemsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupServiceJobsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupScheduleCmd; @@ -80,6 +85,7 @@ import org.apache.cloudstack.api.command.user.backup.repository.UpdateBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ExtractResponse; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupDetailsDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; @@ -95,6 +101,7 @@ import org.apache.cloudstack.poll.BackgroundPollManager; import org.apache.cloudstack.poll.BackgroundPollTask; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; @@ -1900,6 +1907,107 @@ private void checkForPendingBackupJobs(final BackupVO backup) { } } + @Override + public List listBackupFilesystems(long backupId, Long volumeId) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + List filesystems = backupProvider.listBackupFilesystems(backupVO, backupVolumeInfo); + + List dataStoreObjectResponseList = new ArrayList<>(); + for (FilesystemInfoTO fsInfoTO : filesystems) { + DataStoreObjectResponse dataStoreObjectResponse = new DataStoreObjectResponse(fsInfoTO.getName(), false, fsInfoTO.getSize(), null); + dataStoreObjectResponse.setIsFilesystem(true); + dataStoreObjectResponse.setFormat(fsInfoTO.getFilesystem()); + + VolumeVO volume = volumeDao.findByIdIncludingRemoved(fsInfoTO.getVolumeId()); + dataStoreObjectResponse.setVolumeId(volume.getUuid()); + dataStoreObjectResponse.setVolumeName(volume.getName()); + dataStoreObjectResponseList.add(dataStoreObjectResponse); + } + + return dataStoreObjectResponseList; + } + + @Override + public List listBackupFiles(long backupId, Long volumeId, String filesystem, String directory, Boolean isSymlink) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + return backupProvider.listBackupFiles(backupVO, backupVolumeInfo, filesystem, directory, isSymlink); + } + + @Override + public ExtractResponse downloadBackupFile(long backupId, Long volumeId, String filesystem, String file) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + String url = backupProvider.downloadBackupFile(backupVO, backupVolumeInfo, filesystem, file); + + ExtractResponse response = new ExtractResponse(); + if (url == null) { + response.setState(Upload.Status.DOWNLOAD_URL_NOT_CREATED.toString()); + return response; + } + + response.setUrl(url); + response.setName(file.substring(file.lastIndexOf("/") + 1)); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + return response; + } + + private Backup.VolumeInfo validateBackupAndGetVolumeInfo(long backupId, Long volumeId, BackupVO backupVO) { + if (backupVO == null) { + logger.warn("Unable to find backup with ID [{}].", backupId); + throw new InvalidParameterValueException("Unable to find backup with given ID."); + } + + if (backupVO.getStatus() != Backup.Status.BackedUp) { + throw new InvalidParameterValueException(String.format("Backup [%s] is not in the right state to list its filesystems. It should be in [%s] state, but it is in [%s] " + + "state.", backupVO.getUuid(), Backup.Status.BackedUp.name(), backupVO.getStatus().name())); + } + + Long vmId = backupVO.getVmId(); + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + if (vm == null) { + throw new CloudRuntimeException(String.format("Unable to find VM for backup [%s].", backupVO.getUuid())); + } + validateBackupForZone(vm.getDataCenterId()); + + List volumeInfos = backupVO.getBackedUpVolumes(); + if (CollectionUtils.isEmpty(volumeInfos)) { + throw new CloudRuntimeException(String.format("Backup [%s] has no volume info metadata. Unable to list filesystems.", backupVO.getUuid())); + } + + Backup.VolumeInfo backupVolumeInfo = null; + if (volumeId != null) { + VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId); + backupVolumeInfo = getVolumeInfo(volumeInfos, volumeVO.getUuid()); + if (backupVolumeInfo == null) { + throw new CloudRuntimeException(String.format("Failed to find volume [%s] in the backed-up volumes metadata.", volumeVO.getUuid())); + } + } + return backupVolumeInfo; + } + /** * Get the pair: hostIp, datastoreUuid in which to restore the volume, based on the VM to be attached information */ @@ -2069,6 +2177,9 @@ public List> getCommands() { cmdList.add(DownloadValidationScreenshotCmd.class); cmdList.add(ListBackupServiceJobsCmd.class); cmdList.add(FinishBackupChainCmd.class); + cmdList.add(ListBackupFilesystemsCmd.class); + cmdList.add(ListBackupFilesCmd.class); + cmdList.add(DownloadBackupFileCmd.class); return cmdList; } @@ -2676,6 +2787,9 @@ public BackupResponse createBackupResponse(Backup backup, Boolean listVmDetails) if (backup.getToCheckpointId() != null) { response.setToCheckpointId(backup.getToCheckpointId()); } + if (KBOSS_BACKUP_PROVIDER.equals(offering.getProvider())) { + response.setBrowsable(true); + } response.setObjectName("backup"); return response; diff --git a/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileObject.java b/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileObject.java new file mode 100644 index 000000000000..3cf5a6643ef8 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileObject.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.to; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import java.util.UUID; + +public class BackupFileObject implements DataObject { + + private DataTO dataTO; + private DataStore dataStore; + private String name; + + public BackupFileObject(DataTO dataTO, DataStore dataStore) { + this.dataTO = dataTO; + this.dataStore = dataStore; + this.name = UUID.randomUUID().toString(); + } + + @Override + public String toString() { + return String.format("BackupFileObject %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "dataTO", "dataStore")); + } + + @Override + public long getId() { + return 0; + } + + @Override + public String getUri() { + return null; + } + + @Override + public DataTO getTO() { + return dataTO; + } + + @Override + public DataStore getDataStore() { + return dataStore; + } + + @Override + public Long getSize() { + return null; + } + + @Override + public long getPhysicalSize() { + return 0; + } + + @Override + public DataObjectType getType() { + return dataTO.getObjectType(); + } + + @Override + public String getUuid() { + return null; + } + + @Override + public boolean delete() { + return false; + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event) { + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) { + } + + @Override + public void incRefCount() { + } + + @Override + public void decRefCount() { + } + + @Override + public Long getRefCount() { + return null; + } + + @Override + public String getName() { + return name; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileTO.java b/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileTO.java new file mode 100644 index 000000000000..3538a65b37f2 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/to/BackupFileTO.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.to; + +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.hypervisor.Hypervisor; + +public class BackupFileTO implements DataTO { + + private DataStoreTO dataStoreTO; + private Hypervisor.HypervisorType hypervisor; + private String path; + + public BackupFileTO(DataStoreTO dataStoreTO, Hypervisor.HypervisorType hypervisor, String path) { + this.dataStoreTO = dataStoreTO; + this.hypervisor = hypervisor; + this.path = path; + } + + @Override + public DataObjectType getObjectType() { + return DataObjectType.ARCHIVE; + } + + @Override + public DataStoreTO getDataStore() { + return dataStoreTO; + } + + @Override + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisor; + } + + @Override + public String getPath() { + return path; + } + + @Override + public long getId() { + return 0; + } +} diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java index 4ee6d28e3291..e0c5086fb5d3 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java @@ -59,6 +59,7 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.ArrayUtils; public class UploadManagerImpl extends ManagerBase implements UploadManager { @@ -381,6 +382,11 @@ public Answer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLComman } } + String deleteGzipResult = deleteIfGzipFile(cmd); + if (deleteGzipResult != null) { + return new Answer(cmd, false, deleteGzipResult); + } + return new Answer(cmd, true, ""); } @@ -408,6 +414,31 @@ protected void deleteEntitySymlinkRootDirectoryIfNeeded(DeleteEntityDownloadURLC } } + /** + * Deletes the extract file if it is a GZIP. Currently, the GZIP format is only used to download files from backups. This cleans up the files from secondary storage. + * */ + private String deleteIfGzipFile(DeleteEntityDownloadURLCommand cmd) { + String path = cmd.getPath(); + if (!path.substring(path.lastIndexOf(".") + 1).equals(ImageFormat.GZIP.getFileExtension())) { + return null; + } + + try { + String fullPath = String.format("/mnt/SecStorage/%s%s%s", cmd.getParentPath(), File.separator, path); + logger.debug("Deleting temporary gzip file at [{}].", fullPath); + Files.deleteIfExists(Path.of(fullPath)); + File dir = new File(fullPath.substring(0, fullPath.lastIndexOf(File.separator))); + if (dir.exists() && dir.isDirectory() && ArrayUtils.isEmpty(dir.listFiles())) { + dir.delete(); + } + } catch (IOException e) { + String errorString = String.format("Error deleting temporary download file %s.", path); + logger.warn(errorString, e); + return errorString; + } + return null; + } + private String getInstallPath(String jobId) { // TODO Auto-generated method stub return null; diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 775de26103a0..888dec71ccd3 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -478,6 +478,7 @@ "label.backingup": "BackingUp", "label.backup.isolated": "Isolated", "label.backup.attach.restore": "Restore and attach backup volume", +"label.backup.files": "Browse backup files", "label.backup.configure.schedule": "Configure Backup Schedule", "label.backup.chain.finish": "Finish backup chain", "label.backupchainsize": "Backup chain size", @@ -2393,6 +2394,7 @@ "label.select.a.zone": "Select a Zone", "label.select.backup.offering": "Select Backup Offering", "label.select.deployment.infrastructure": "Select deployment infrastructure", +"label.select.filesystem": "Select filesystem", "label.select.guest.os.type": "Please select the guest OS type", "label.select.network": "Select Network", "label.select.period": "Select period", @@ -2512,6 +2514,7 @@ "label.storageaccessgroups": "Storage Access Groups", "label.storageallocated": "Allocated size", "label.storagetotal": "Total size", +"label.symlink": "Symbolic link?", "label.storageused": "Used size", "label.clusterstorageaccessgroups": "Cluster Storage Access Groups", "label.podstorageaccessgroups": "Pod Storage Access Groups", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index b3eae6eb11ce..cf91f79daa9e 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -441,6 +441,7 @@ "label.backedup": "Salvo", "label.backingup": "Salvando", "label.backup.attach.restore": "Restaurar e anexar volume de backup", +"label.backup.files": "Navegar arquivos de backup", "label.backuplimit": "Limite de backups", "label.backup.storage": "Armazenamento de backup", "label.backupstoragelimit": "Limite de armazenamento de backup (GiB)", @@ -2110,6 +2111,7 @@ "label.select.2fa.provider": "Selecione o provedor", "label.select.a.zone": "Selecione uma zona", "label.select.deployment.infrastructure": "Selecione uma infraestrutura de implanta\u00e7\u00e3o", +"label.select.filesystem": "Selecione o filesystem", "label.select.guest.os.type": "Por favor, selecione o tipo de SO convidado", "label.select.network": "Selecionar Rede", "label.select.period": "Selecionar per\u00edodo", @@ -2297,6 +2299,7 @@ "label.supportsvmautoscaling": "Suporta escalonamento autom\u00e1tico", "label.suspend.project": "Suspender projeto", "label.switch.type": "Tipo de switch", +"label.symlink": "Link simb\u00f3lico?", "label.sync.storage": "Sincronizar pool do armazenamento", "label.system.ip.pool": "Pool do Sistema", "label.system.offering": "Ofertas de sistema", diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 702b782690af..3d306b420136 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -507,6 +507,13 @@ export default { { name: 'instance.metadata', component: shallowRef(defineAsyncComponent(() => import('@/components/view/BackupMetadata.vue'))) + }, + { + name: 'backup.files', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/BackupFileBrowser.vue'))), + show: (resource, route, userInfo) => { + return resource.browsable ?? false + } } ], actions: [ diff --git a/ui/src/core/lazy_lib/icons_use.js b/ui/src/core/lazy_lib/icons_use.js index 43c6f822de31..7ca4f64375ad 100644 --- a/ui/src/core/lazy_lib/icons_use.js +++ b/ui/src/core/lazy_lib/icons_use.js @@ -81,6 +81,7 @@ import { EyeInvisibleOutlined, EyeOutlined, FieldTimeOutlined, + FileOutlined, FileDoneOutlined, FileProtectOutlined, FileSyncOutlined, @@ -257,6 +258,7 @@ export default { app.component('EyeInvisibleOutlined', EyeInvisibleOutlined) app.component('EyeOutlined', EyeOutlined) app.component('FieldTimeOutlined', FieldTimeOutlined) + app.component('FileOutlined', FileOutlined) app.component('FileDoneOutlined', FileDoneOutlined) app.component('FileProtectOutlined', FileProtectOutlined) app.component('FileSyncOutlined', FileSyncOutlined) diff --git a/ui/src/views/infra/BackupFileBrowser.vue b/ui/src/views/infra/BackupFileBrowser.vue new file mode 100644 index 000000000000..35ae0393f5e1 --- /dev/null +++ b/ui/src/views/infra/BackupFileBrowser.vue @@ -0,0 +1,330 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + +