add global setting to allow parallel execution on vmware

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -20,7 +20,6 @@
 import java.util.Map;
 
 import org.apache.cloudstack.backup.Backup;
-import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.to.NicTO;
@@ -35,8 +34,7 @@
 import com.cloud.vm.VirtualMachineProfile;
 
 public interface HypervisorGuru extends Adapter {
-    ConfigKey<Boolean> VmwareFullClone = new ConfigKey<Boolean>("Advanced", Boolean.class, "vmware.create.full.clone", "true",
-            "If set to true, creates guest VMs as full clones on ESX", false);
+
     HypervisorType getHypervisorType();
 
     /**

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -40,57 +40,86 @@ public interface CapacityManager {
     static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold";
     static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor";
     static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold";
-    static final String VmwareCreateCloneFullCK = "vmware.create.full.clone";
-
-    static final ConfigKey<Float> CpuOverprovisioningFactor = new ConfigKey<Float>(Float.class, CpuOverprovisioningFactorCK, "Advanced", "1.0",
-        "Used for CPU overprovisioning calculation; available CPU will be (actualCpuCapacity * cpu.overprovisioning.factor)", true, ConfigKey.Scope.Cluster, null);
-    static final ConfigKey<Float> MemOverprovisioningFactor = new ConfigKey<Float>(Float.class, MemOverprovisioningFactorCK, "Advanced", "1.0",
-        "Used for memory overprovisioning calculation", true, ConfigKey.Scope.Cluster, null);
-    static final ConfigKey<Double> StorageCapacityDisableThreshold = new ConfigKey<Double>("Alert", Double.class, StorageCapacityDisableThresholdCK, "0.85",
-        "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true,
-        ConfigKey.Scope.Zone);
-    static final ConfigKey<Double> StorageOverprovisioningFactor = new ConfigKey<Double>("Storage", Double.class, StorageOverprovisioningFactorCK, "2",
-        "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.StoragePool);
+
+    static final String CATEGORY_ADVANCED = "Advanced";
+    static final String CATEGORY_ALERT = "Alert";
+
+    static final ConfigKey<Float> CpuOverprovisioningFactor =
+            new ConfigKey<>(
+                    Float.class,
+                    CpuOverprovisioningFactorCK,
+                    CATEGORY_ADVANCED,
+                    "1.0",
+                    "Used for CPU overprovisioning calculation; available CPU will be (actualCpuCapacity * cpu.overprovisioning.factor)",
+                    true,
+                    ConfigKey.Scope.Cluster,
+                    null);
+    static final ConfigKey<Float> MemOverprovisioningFactor =
+            new ConfigKey<>(
+                    Float.class,
+                    MemOverprovisioningFactorCK,
+                    CATEGORY_ADVANCED,
+                    "1.0",
+                    "Used for memory overprovisioning calculation",
+                    true,
+                    ConfigKey.Scope.Cluster,
+                    null);
+    static final ConfigKey<Double> StorageCapacityDisableThreshold =
+            new ConfigKey<>(
+                    CATEGORY_ALERT,
+                    Double.class,
+                    StorageCapacityDisableThresholdCK,
+                    "0.85",
+                    "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.",
+                    true,
+                    ConfigKey.Scope.Zone);
+    static final ConfigKey<Double> StorageOverprovisioningFactor =
+            new ConfigKey<>(
+                    "Storage",
+                    Double.class,
+                    StorageOverprovisioningFactorCK,
+                    "2",
+                    "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)",
+                    true,
+                    ConfigKey.Scope.StoragePool);
     static final ConfigKey<Double> StorageAllocatedCapacityDisableThreshold =
-        new ConfigKey<Double>(
-            "Alert",
-            Double.class,
-            StorageAllocatedCapacityDisableThresholdCK,
-            "0.85",
-            "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.",
-            true, ConfigKey.Scope.Zone);
+            new ConfigKey<>(
+                    CATEGORY_ALERT,
+                    Double.class,
+                    StorageAllocatedCapacityDisableThresholdCK,
+                    "0.85",
+                    "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.",
+                    true,
+                    ConfigKey.Scope.Zone);
     static final ConfigKey<Boolean> StorageOperationsExcludeCluster =
-            new ConfigKey<Boolean>(
+            new ConfigKey<>(
                     Boolean.class,
                     "cluster.storage.operations.exclude",
-                    "Advanced",
+                    CATEGORY_ADVANCED,
                     "false",
                     "Exclude cluster from storage operations",
                     true,
                     ConfigKey.Scope.Cluster,
                     null);
-    static final ConfigKey<Boolean> VmwareCreateCloneFull =
-            new ConfigKey<Boolean>(
-                    "Storage",
-                    Boolean.class,
-                    VmwareCreateCloneFullCK,
-                    "false",
-                    "If set to true, creates VMs as full clones on ESX hypervisor",
-                    true,
-                    ConfigKey.Scope.StoragePool);
     static final ConfigKey<String> ImageStoreNFSVersion =
-            new ConfigKey<String>(
+            new ConfigKey<>(
                     String.class,
                     "secstorage.nfs.version",
-                    "Advanced",
+                    CATEGORY_ADVANCED,
                     null,
                     "Enforces specific NFS version when mounting Secondary Storage. If NULL default selection is performed",
                     true,
                     ConfigKey.Scope.ImageStore,
                     null);
 
-    static final ConfigKey<Float> SecondaryStorageCapacityThreshold = new ConfigKey<Float>("Advanced", Float.class, "secondary.storage.capacity.threshold", "0.90",
-            "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.", true);
+    static final ConfigKey<Float> SecondaryStorageCapacityThreshold =
+            new ConfigKey<>(
+                    CATEGORY_ADVANCED,
+                    Float.class,
+                    "secondary.storage.capacity.threshold",
+                    "0.90",
+                    "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.",
+                    true);
 
     public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);
 

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -163,6 +163,44 @@ public interface StorageManager extends StorageService {
             true,
             ConfigKey.Scope.Cluster,
             null);
+    ConfigKey<Boolean> VmwareCreateCloneFull = new ConfigKey<>(Boolean.class,
+            "vmware.create.full.clone",
+            "Storage",
+            "false",
+            "If set to true, creates VMs as full clones on ESX hypervisor",
+            true,
+            ConfigKey.Scope.StoragePool,
+            null);
+    ConfigKey<Boolean> VmwareAllowParallelExecution = new ConfigKey<>(Boolean.class,
+            "vmware.allow.parallel.command.execution",
+            "Advanced",
+            "false",
+            "allow commands to be executed in parallel in spite of 'vmware.create.full.clone' being set to true.",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
+    /**
+     * should we execute in sequence not involving any storages?
+     * @return tru if commands should execute in sequence
+     */
+    static boolean shouldExecuteInSequenceOnVmware() {
+        return shouldExecuteInSequenceOnVmware(null, null);
+    }
+
+    static boolean shouldExecuteInSequenceOnVmware(Long srcStoreId, Long dstStoreId) {
+        final Boolean fullClone = getFullCloneConfiguration(srcStoreId) || getFullCloneConfiguration(dstStoreId);
+        final Boolean allowParallel = getAllowParallelExecutionConfiguration();
+        return fullClone && !allowParallel;
+    }
+
+    static Boolean getAllowParallelExecutionConfiguration() {
+        return VmwareAllowParallelExecution.value();
+    }
+
+    static Boolean getFullCloneConfiguration(Long storeId) {
+        return VmwareCreateCloneFull.valueIn(storeId);
+    }
 
     /**
      * Returns a comma separated list of tags for the specified storage pool

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1557,8 +1557,7 @@ public boolean getExecuteInSequence(final HypervisorType hypervisorType) {
             case LXC:
                 return false;
             case VMware:
-                final Boolean fullClone = HypervisorGuru.VmwareFullClone.value();
-                return fullClone;
+                return StorageManager.shouldExecuteInSequenceOnVmware();
             default:
                 return ExecuteInSequence.value();
         }

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -85,7 +85,6 @@
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.agent.manager.allocator.PodAllocator;
-import com.cloud.capacity.CapacityManager;
 import com.cloud.cluster.ClusterManager;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.dc.DataCenter;
@@ -1704,7 +1703,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
             if (vm.getHypervisorType().equals(HypervisorType.VMware)) {
                 // retrieve clone flag.
                 UserVmCloneType cloneType = UserVmCloneType.linked;
-                Boolean value = CapacityManager.VmwareCreateCloneFull.valueIn(vol.getPoolId());
+                Boolean value = StorageManager.VmwareCreateCloneFull.valueIn(vol.getPoolId());
                 if (value != null && value) {
                     cloneType = UserVmCloneType.full;
                 }

engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java

@@ -32,6 +32,7 @@
 import java.util.Map;
 
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.storage.StorageManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
@@ -56,7 +57,6 @@
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.host.HostVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.hypervisor.HypervisorGuru;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.DiskOfferingVO;
@@ -209,9 +209,23 @@ public void testSendStopWithNullAnswer() throws Exception {
     public void testExeceuteInSequence() {
         assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.XenServer) == false);
         assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.KVM) == false);
-        assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware) == HypervisorGuru.VmwareFullClone.value());
         assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.Ovm3) == VirtualMachineManager.ExecuteInSequence.value());
     }
+    @Test
+    public void testExeceuteInSequenceVmware() {
+        when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.FALSE);
+        when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.FALSE);
+        assertFalse("no full clones so no need to execute in sequence", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware));
+        when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.TRUE);
+        when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.FALSE);
+        assertTrue("full clones and no explicit parallel execution allowed, should execute in sequence", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware));
+        when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.TRUE);
+        when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.TRUE);
+        assertFalse("execute in sequence should not be needed as parallel is allowed", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware));
+        when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.FALSE);
+        when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.TRUE);
+        assertFalse("double reasons to allow parallel execution", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware));
+    }
 
     @Test
     public void testCheckIfCanUpgrade() throws Exception {

engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java

@@ -47,10 +47,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
 
     int getOtherPersistentNetworksCount(long id, String broadcastURI, boolean isPersistent);
 
-    @Override
-    @Deprecated
-    NetworkVO persist(NetworkVO vo);
-
     /**
      * Retrieves the next available mac address in this network configuration.
      *

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -62,7 +62,6 @@
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
-import com.cloud.capacity.CapacityManager;
 import com.cloud.configuration.Config;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor;
@@ -226,7 +225,7 @@ protected DataTO addFullCloneAndDiskprovisiongStrictnessFlagOnVMwareDest(DataTO
             DataStoreTO dataStoreTO = dataTO.getDataStore();
             if (dataStoreTO != null && dataStoreTO instanceof PrimaryDataStoreTO){
                 PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) dataStoreTO;
-                primaryDataStoreTO.setFullCloneFlag(CapacityManager.VmwareCreateCloneFull.valueIn(primaryDataStoreTO.getId()));
+                primaryDataStoreTO.setFullCloneFlag(StorageManager.VmwareCreateCloneFull.valueIn(primaryDataStoreTO.getId()));
                 StoragePool pool = storageManager.getStoragePool(primaryDataStoreTO.getId());
                 primaryDataStoreTO.setDiskProvisioningStrictnessFlag(storageManager.DiskProvisioningStrictness.valueIn(pool.getDataCenterId()));
             }

engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java

@@ -82,7 +82,7 @@ public void setup() throws Exception {
     }
 
     private void replaceVmwareCreateCloneFullField() throws Exception {
-        Field field = CapacityManager.class.getDeclaredField("VmwareCreateCloneFull");
+        Field field = StorageManager.class.getDeclaredField("VmwareCreateCloneFull");
         field.setAccessible(true);
         // remove final modifier from field
         Field modifiersField = Field.class.getDeclaredField("modifiers");

plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalGuru.java

@@ -30,24 +30,18 @@
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.to.VirtualMachineTO;
-import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.hypervisor.HypervisorGuru;
 import com.cloud.hypervisor.HypervisorGuruBase;
 import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachineProfile;
-import com.cloud.vm.dao.VMInstanceDao;
 
 public class BareMetalGuru extends HypervisorGuruBase implements HypervisorGuru {
     private static final Logger s_logger = Logger.getLogger(BareMetalGuru.class);
     @Inject
     GuestOSDao _guestOsDao;
-    @Inject
-    HostDao _hostDao;
-    @Inject
-    VMInstanceDao _vmDao;
 
     protected BareMetalGuru() {
         super();
@@ -62,7 +56,7 @@ public HypervisorType getHypervisorType() {
     public VirtualMachineTO implement(VirtualMachineProfile vm) {
         VirtualMachineTO to = toVirtualMachineTO(vm);
 
-        VMInstanceVO vo = _vmDao.findById(vm.getId());
+        VMInstanceVO vo = virtualMachineDao.findById(vm.getId());
         if (vo.getLastHostId() == null) {
             to.setBootArgs(BaremetalManager.DO_PXE);
         }

plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/guru/HypervGuru.java

@@ -34,7 +34,6 @@
 import com.cloud.network.NetworkModel;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.TrafficType;
-import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
 import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.dao.GuestOSDao;
@@ -51,7 +50,6 @@ public class HypervGuru extends HypervisorGuruBase implements HypervisorGuru {
     @Inject
     private GuestOSDao _guestOsDao;
     @Inject HypervManager _hypervMgr;
-    @Inject NetworkDao _networkDao;
     @Inject NetworkModel _networkMgr;
     int MaxNicSupported = 8;
     @Override
@@ -109,7 +107,7 @@ else if (nicProfile.getTrafficType() == TrafficType.Control) {
                     profile = controlNicProfile;
                 }
 
-                NetworkVO network = _networkDao.findById(networkId);
+                NetworkVO network = networkDao.findById(networkId);
                 // for Hyperv Hot Nic plug is not supported and it will support upto 8 nics.
                 // creating the VR with extra nics (actual nics(3) + extra nics) will be 8
                 for(; i < MaxNicSupported; i++) {