Merge branch '4.19'

agent/src/main/java/com/cloud/agent/Agent.java

@@ -1127,6 +1127,12 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                     logger.error("Error parsing task", e);
                 }
             } else if (task.getType() == Task.Type.DISCONNECT) {
+                try {
+                    // an issue has been found if reconnect immediately after disconnecting. please refer to https://github.com/apache/cloudstack/issues/8517
+                    // wait 5 seconds before reconnecting
+                    Thread.sleep(5000);
+                } catch (InterruptedException e) {
+                }
                 reconnect(task.getLink());
                 return;
             } else if (task.getType() == Task.Type.OTHER) {

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -751,7 +751,7 @@ public class AgentProperties{
     public static final Property<Integer> IOTHREADS = new Property<>("iothreads", 1);
 
     /**
-     * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM
+     * Enable verbose mode for virt-v2v Instance Conversion from VMware to KVM
      * Data type: Boolean.<br>
      * Default value: <code>false</code>
      */

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -18,40 +18,39 @@
  */
 package com.cloud.agent.api.to;
 
+import java.io.Serializable;
+
 import com.cloud.agent.api.LogLevel;
 import com.cloud.hypervisor.Hypervisor;
 
-import java.io.Serializable;
-
 public class RemoteInstanceTO implements Serializable {
 
     private Hypervisor.HypervisorType hypervisorType;
-    private String hostName;
     private String instanceName;
 
-    // Vmware Remote Instances parameters
+    // VMware Remote Instances parameters (required for exporting OVA through ovftool)
     // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
     private String vcenterUsername;
     @LogLevel(LogLevel.Log4jLevel.Off)
     private String vcenterPassword;
     private String vcenterHost;
     private String datacenterName;
-    private String clusterName;
 
     public RemoteInstanceTO() {
     }
 
-    public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost,
+    public RemoteInstanceTO(String instanceName) {
-                            String datacenterName, String clusterName,
+        this.hypervisorType = Hypervisor.HypervisorType.VMware;
-                            String vcenterUsername, String vcenterPassword) {
+        this.instanceName = instanceName;
+    }
+
+    public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
-        this.hostName = hostName;
         this.instanceName = instanceName;
         this.vcenterHost = vcenterHost;
-        this.datacenterName = datacenterName;
-        this.clusterName = clusterName;
         this.vcenterUsername = vcenterUsername;
         this.vcenterPassword = vcenterPassword;
+        this.datacenterName = datacenterName;
     }
 
     public Hypervisor.HypervisorType getHypervisorType() {
@@ -62,10 +61,6 @@ public class RemoteInstanceTO implements Serializable {
         return this.instanceName;
     }
 
-    public String getHostName() {
-        return this.hostName;
-    }
-
     public String getVcenterUsername() {
         return vcenterUsername;
     }
@@ -81,8 +76,4 @@ public class RemoteInstanceTO implements Serializable {
     public String getDatacenterName() {
         return datacenterName;
     }
-
-    public String getClusterName() {
-        return clusterName;
-    }
 }

api/src/main/java/com/cloud/host/Host.java

@@ -54,6 +54,7 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
     }
     public static final String HOST_UEFI_ENABLE = "host.uefi.enable";
     public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
 
     /**
      * @return name of the machine.

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -23,6 +23,7 @@ import org.apache.cloudstack.backup.Backup;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -101,21 +102,20 @@ public interface HypervisorGuru extends Adapter {
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *
      * @param vm the stopped vm to migrate
-     * @param destination the primary storage pool to migrate to
+     * @param volumeToPool the primary storage pools to migrate to
      * @return a list of commands to perform for a successful migration
      */
     List<Command> finalizeMigrate(VirtualMachine vm, Map<Volume, StoragePool> volumeToPool);
 
 
     /**
-     * Will perform a clone of a VM on an external host (if the guru can handle)
+     * Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle)
      * @param hostIp VM's source host IP
-     * @param vmName name of the source VM to clone from
+     * @param vmName name of the source VM (clone VM name if cloned)
      * @param params hypervisor specific additional parameters
-     * @return a reference to the cloned VM
+     * @return a reference to the hypervisor or cloned VM, and cloned flag
      */
-    UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName,
+    Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params);
-                                                   Map<String, String> params);
 
     /**
      * Removes a VM created as a clone of a VM on an external host
@@ -124,6 +124,23 @@ public interface HypervisorGuru extends Adapter {
      * @param params hypervisor specific additional parameters
      * @return true if the operation succeeds, false if not
      */
-    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName,
+    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params);
-                                              Map<String, String> params);
+
+    /**
+     * Create an OVA/OVF template of a VM on an external host (if the guru can handle)
+     * @param hostIp VM's source host IP
+     * @param vmName name of the source VM to create template from
+     * @param params hypervisor specific additional parameters
+     * @param templateLocation datastore to create the template file
+     * @return the created template dir/name
+     */
+    String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf);
+
+    /**
+     * Removes the template on the location
+     * @param templateLocation datastore to remove the template file
+     * @param templateDir the template dir to remove from datastore
+     * @return true if the operation succeeds, false if not
+     */
+    boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir);
 }

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -317,6 +317,8 @@ public interface NetworkModel {
 
     void checkIp6Parameters(String startIPv6, String endIPv6, String ip6Gateway, String ip6Cidr) throws InvalidParameterValueException;
 
+    void checkIp6CidrSizeEqualTo64(String ip6Cidr) throws InvalidParameterValueException;
+
     void checkRequestedIpAddresses(long networkId, IpAddresses ips) throws InvalidParameterValueException;
 
     String getStartIpv6Address(long id);

api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java

@@ -17,17 +17,22 @@
 package com.cloud.network;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
 
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.router.VirtualRouter;
 import com.cloud.user.Account;
 import com.cloud.utils.Pair;
 import com.cloud.vm.Nic;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
 
 public interface VirtualNetworkApplianceService {
     /**
@@ -62,6 +67,10 @@ public interface VirtualNetworkApplianceService {
 
     VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException;
 
+    void startRouterForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params, DeploymentPlanner planner)
+            throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
+            OperationTimedoutException;
+
     VirtualRouter destroyRouter(long routerId, Account caller, Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException;
 
     VirtualRouter findRouter(long routerId);

api/src/main/java/com/cloud/network/guru/NetworkGuru.java

@@ -212,4 +212,7 @@ public interface NetworkGuru extends Adapter {
 
     boolean isMyTrafficType(TrafficType type);
 
+    default boolean isSlaacV6Only() {
+        return true;
+    }
 }

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -62,6 +62,7 @@ public class NicProfile implements InternalIdentity, Serializable {
     String iPv4Dns1;
     String iPv4Dns2;
     String requestedIPv4;
+    boolean ipv4AllocationRaceCheck;
 
     // IPv6
     String iPv6Address;
@@ -405,6 +406,13 @@ public class NicProfile implements InternalIdentity, Serializable {
         this.mtu = mtu;
     }
 
+    public boolean getIpv4AllocationRaceCheck() {
+        return this.ipv4AllocationRaceCheck;
+    }
+
+    public void setIpv4AllocationRaceCheck(boolean ipv4AllocationRaceCheck) {
+        this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck;
+    }
 
     //
     // OTHER METHODS

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -42,6 +42,7 @@ import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd;
 import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd;
 
 import com.cloud.dc.DataCenter;
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ManagementServerException;
@@ -112,6 +113,10 @@ public interface UserVmService {
 
     void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException;
 
+    void startVirtualMachineForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+          DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException;
+
     UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException;
 
     /**

api/src/main/java/com/cloud/vm/VirtualMachineProfile.java

@@ -192,6 +192,10 @@ public interface VirtualMachineProfile {
 
     Map<Param, Object> getParameters();
 
+    void setCpuOvercommitRatio(Float cpuOvercommitRatio);
+
+    void setMemoryOvercommitRatio(Float memoryOvercommitRatio);
+
     Float getCpuOvercommitRatio();
 
     Float getMemoryOvercommitRatio();

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -191,6 +191,7 @@ public class ApiConstants {
     public static final String FORCED = "forced";
     public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage";
     public static final String FORCE_DELETE_HOST = "forcedeletehost";
+    public static final String FORCE_MS_TO_IMPORT_VM_FILES = "forcemstoimportvmfiles";
     public static final String FORMAT = "format";
     public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork";
     public static final String FOR_SYSTEM_VMS = "forsystemvms";
@@ -239,6 +240,7 @@ public class ApiConstants {
     public static final String NEXT_ACL_RULE_ID = "nextaclruleid";
     public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash";
     public static final String IMAGE_PATH = "imagepath";
+    public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported";
     public static final String INTERNAL_DNS1 = "internaldns1";
     public static final String INTERNAL_DNS2 = "internaldns2";
     public static final String INTERNET_PROTOCOL = "internetprotocol";
@@ -1135,6 +1137,8 @@ public class ApiConstants {
     public static final String WEBHOOK_ID = "webhookid";
     public static final String WEBHOOK_NAME = "webhookname";
 
+    public static final String NFS_MOUNT_OPTIONS = "nfsmountopts";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java

@@ -37,6 +37,7 @@ import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.VmwareDatacenterResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.vm.VmImportService;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 
@@ -116,40 +117,44 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
             description = "Temp Path on external host for disk image copy" )
     private String tmpPath;
 
-    // Import from Vmware to KVM migration parameters
+    // Import from VMware to KVM migration parameters
 
     @Parameter(name = ApiConstants.EXISTING_VCENTER_ID,
             type = CommandType.UUID,
             entityType = VmwareDatacenterResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter")
+            description = "(only for importing VMs from VMware to KVM) UUID of a linked existing vCenter")
     private Long existingVcenterId;
 
     @Parameter(name = ApiConstants.HOST_IP,
             type = BaseCmd.CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.")
+            description = "(only for importing VMs from VMware to KVM) VMware ESXi host IP/Name.")
     private String hostip;
 
     @Parameter(name = ApiConstants.VCENTER,
             type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.")
+            description = "(only for importing VMs from VMware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.")
     private String vcenter;
 
     @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.")
+            description = "(only for importing VMs from VMware to KVM) Name of VMware datacenter.")
     private String datacenterName;
 
     @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.")
+            description = "(only for importing VMs from VMware to KVM) Name of VMware cluster.")
     private String clusterName;
 
     @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.")
     private Long convertInstanceHostId;
 
     @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
     private Long convertStoragePoolId;
 
+    @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN,
+            description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.")
+    private Boolean forceMsToImportVmFiles;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -198,6 +203,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
         return convertStoragePoolId;
     }
 
+    public Boolean getForceMsToImportVmFiles() {
+        return BooleanUtils.toBooleanDefaultIfNull(forceMsToImportVmFiles, false);
+    }
+
     public String getHypervisor() {
         return hypervisor;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java

@@ -72,7 +72,7 @@ public class AssignToLoadBalancerRuleCmd extends BaseAsyncCmd {
 
     @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID_IP,
             type = CommandType.MAP,
-            description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].ip=10.1.1.75",
+            description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].vmip=10.1.1.75",
             since = "4.4")
     private Map vmIdIpMap;
 

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -29,6 +29,7 @@ import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
 
 import com.cloud.host.Host;
 import com.cloud.host.Status;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.serializer.Param;
 import com.google.gson.annotations.SerializedName;
 
@@ -285,6 +286,10 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "true if the host supports encryption", since = "4.18")
     private Boolean encryptionSupported;
 
+    @SerializedName(ApiConstants.INSTANCE_CONVERSION_SUPPORTED)
+    @Param(description = "true if the host supports instance conversion (using virt-v2v)", since = "4.19.1")
+    private Boolean instanceConversionSupported;
+
     @Override
     public String getObjectId() {
         return this.getId();
@@ -550,7 +555,7 @@ public class HostResponse extends BaseResponseWithAnnotations {
         this.username = username;
     }
 
-    public void setDetails(Map details) {
+    public void setDetails(Map details, Hypervisor.HypervisorType hypervisorType) {
 
         if (details == null) {
             return;
@@ -571,6 +576,15 @@ public class HostResponse extends BaseResponseWithAnnotations {
             this.setEncryptionSupported(new Boolean(false)); // default
         }
 
+        if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) {
+            if (detailsCopy.containsKey(Host.HOST_INSTANCE_CONVERSION)) {
+                this.setInstanceConversionSupported(Boolean.parseBoolean((String) detailsCopy.get(Host.HOST_INSTANCE_CONVERSION)));
+                detailsCopy.remove(Host.HOST_INSTANCE_CONVERSION);
+            } else {
+                this.setInstanceConversionSupported(new Boolean(false)); // default
+            }
+        }
+
         this.details = detailsCopy;
     }
 
@@ -761,6 +775,10 @@ public class HostResponse extends BaseResponseWithAnnotations {
         this.encryptionSupported = encryptionSupported;
     }
 
+    public void setInstanceConversionSupported(Boolean instanceConversionSupported) {
+        this.instanceConversionSupported = instanceConversionSupported;
+    }
+
     public Boolean getIsTagARule() {
         return isTagARule;
     }

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "the tags for the storage pool")
     private String tags;
 
+    @SerializedName(ApiConstants.NFS_MOUNT_OPTIONS)
+    @Param(description = "the nfs mount options for the storage pool", since = "4.19.1")
+    private String nfsMountOpts;
+
     @SerializedName(ApiConstants.IS_TAG_A_RULE)
     @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
     private Boolean isTagARule;
@@ -347,4 +351,12 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     public void setProvider(String provider) {
         this.provider = provider;
     }
+
+    public String getNfsMountOpts() {
+        return nfsMountOpts;
+    }
+
+    public void setNfsMountOpts(String nfsMountOpts) {
+        this.nfsMountOpts = nfsMountOpts;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -17,6 +17,8 @@
 
 package org.apache.cloudstack.vm;
 
+import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
+
 import java.util.List;
 
 public class UnmanagedInstanceTO {
@@ -317,6 +319,16 @@ public class UnmanagedInstanceTO {
         public int getDatastorePort() {
             return datastorePort;
         }
+
+        @Override
+        public String toString() {
+            return "Disk {" +
+                    "diskId='" + diskId + '\'' +
+                    ", capacity=" + toHumanReadableSize(capacity) +
+                    ", controller='" + controller + '\'' +
+                    ", controllerUnit=" + controllerUnit +
+                    "}";
+        }
     }
 
     public static class Nic {
@@ -409,5 +421,14 @@ public class UnmanagedInstanceTO {
         public void setPciSlot(String pciSlot) {
             this.pciSlot = pciSlot;
         }
+
+        @Override
+        public String toString() {
+            return "Nic{" +
+                    "nicId='" + nicId + '\'' +
+                    ", adapterType='" + adapterType + '\'' +
+                    ", macAddress='" + macAddress + '\'' +
+                    "}";
+        }
     }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java

@@ -39,6 +39,37 @@ public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService,
             ConfigKey.Scope.Global,
             null);
 
+    ConfigKey<Integer> ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class,
+            "convert.vmware.instance.to.kvm.timeout",
+            "Advanced",
+            "3",
+            "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
+    ConfigKey<Integer> ThreadsOnMSToImportVMwareVMFiles = new ConfigKey<>(Integer.class,
+            "threads.on.ms.to.import.vmware.vm.files",
+            "Advanced",
+            "0",
+            "Threads to use on the management server when importing VM files from VMWare." +
+                    " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." +
+                    " Max number is 10, Default is 0.",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
+    ConfigKey<Integer> ThreadsOnKVMHostToImportVMwareVMFiles = new ConfigKey<>(Integer.class,
+            "threads.on.kvm.host.to.import.vmware.vm.files",
+            "Advanced",
+            "0",
+            "Threads to use on the KVM host (by the ovftool, if the version is 4.4.0+) when importing VM files from VMWare." +
+                    " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." +
+                    " Max number is 10, Default is 0.",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
     static boolean isSupported(Hypervisor.HypervisorType hypervisorType) {
         return hypervisorType == VMware || hypervisorType == KVM;
     }

api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java

@@ -23,6 +23,8 @@ import org.junit.Test;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.cloud.hypervisor.Hypervisor;
+
 public final class HostResponseTest extends TestCase {
 
     private static final String VALID_KEY = "validkey";
@@ -32,7 +34,7 @@ public final class HostResponseTest extends TestCase {
     public void testSetDetailsNull() {
 
         final HostResponse hostResponse = new HostResponse();
-        hostResponse.setDetails(null);
+        hostResponse.setDetails(null, null);
 
         assertEquals(null, hostResponse.getDetails());
 
@@ -51,7 +53,7 @@ public final class HostResponseTest extends TestCase {
         final Map expectedDetails = new HashedMap();
         expectedDetails.put(VALID_KEY, VALID_VALUE);
 
-        hostResponse.setDetails(details);
+        hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM);
         final Map actualDetails = hostResponse.getDetails();
 
         assertTrue(details != actualDetails);
@@ -70,7 +72,7 @@ public final class HostResponseTest extends TestCase {
         final Map expectedDetails = new HashedMap();
         expectedDetails.put(VALID_KEY, VALID_VALUE);
 
-        hostResponse.setDetails(details);
+        hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM);
         final Map actualDetails = hostResponse.getDetails();
 
         assertTrue(details != actualDetails);

core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+public class CheckConvertInstanceAnswer extends Answer {
+
+    private boolean ovfExportSupported = false;
+
+    public CheckConvertInstanceAnswer() {
+        super();
+    }
+
+    public CheckConvertInstanceAnswer(Command command, boolean success) {
+        super(command, success, "");
+    }
+
+    public CheckConvertInstanceAnswer(Command command, boolean success, String details) {
+        super(command, success, details);
+    }
+
+    public CheckConvertInstanceAnswer(Command command, boolean success, boolean ovfExportSupported, String details) {
+        super(command, success, details);
+        this.ovfExportSupported = ovfExportSupported;
+    }
+
+    public boolean isOvfExportSupported() {
+        return ovfExportSupported;
+    }
+}

core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java

@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+public class CheckConvertInstanceCommand extends Command {
+    boolean checkWindowsGuestConversionSupport = false;
+
+    public CheckConvertInstanceCommand() {
+    }
+
+    public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport) {
+        this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public boolean getCheckWindowsGuestConversionSupport() {
+        return checkWindowsGuestConversionSupport;
+    }
+}

core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java

@@ -28,16 +28,24 @@ public class ConvertInstanceCommand extends Command {
     private Hypervisor.HypervisorType destinationHypervisorType;
     private List<String> destinationStoragePools;
     private DataStoreTO conversionTemporaryLocation;
+    private String templateDirOnConversionLocation;
+    private boolean checkConversionSupport;
+    private boolean exportOvfToConversionLocation;
+    private int threadsCountToExportOvf = 0;
 
     public ConvertInstanceCommand() {
     }
 
     public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType,
-                                  List<String> destinationStoragePools, DataStoreTO conversionTemporaryLocation) {
+                                  List<String> destinationStoragePools, DataStoreTO conversionTemporaryLocation,
+                                  String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) {
         this.sourceInstance = sourceInstance;
         this.destinationHypervisorType = destinationHypervisorType;
         this.destinationStoragePools = destinationStoragePools;
         this.conversionTemporaryLocation = conversionTemporaryLocation;
+        this.templateDirOnConversionLocation = templateDirOnConversionLocation;
+        this.checkConversionSupport = checkConversionSupport;
+        this.exportOvfToConversionLocation = exportOvfToConversionLocation;
     }
 
     public RemoteInstanceTO getSourceInstance() {
@@ -56,6 +64,26 @@ public class ConvertInstanceCommand extends Command {
         return conversionTemporaryLocation;
     }
 
+    public String getTemplateDirOnConversionLocation() {
+        return templateDirOnConversionLocation;
+    }
+
+    public boolean getCheckConversionSupport() {
+        return checkConversionSupport;
+    }
+
+    public boolean getExportOvfToConversionLocation() {
+        return exportOvfToConversionLocation;
+    }
+
+    public int getThreadsCountToExportOvf() {
+        return threadsCountToExportOvf;
+    }
+
+    public void setThreadsCountToExportOvf(int threadsCountToExportOvf) {
+        this.threadsCountToExportOvf = threadsCountToExportOvf;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;

core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java

@@ -46,6 +46,10 @@ public class ModifyStoragePoolCommand extends Command {
         this.details = details;
     }
 
+    public ModifyStoragePoolCommand(boolean add, StoragePool pool, Map<String, String> details) {
+        this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes()), details);
+    }
+
     public ModifyStoragePoolCommand(boolean add, StoragePool pool) {
         this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes()));
     }

engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java

@@ -126,6 +126,8 @@ public interface ResourceManager extends ResourceService, Configurable {
 
     public List<HostVO> listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType type, long dcId);
 
+    public List<HostVO> listAllUpHostsInOneZoneByHypervisor(HypervisorType type, long dcId);
+
     public List<HostVO> listAllUpAndEnabledHostsInOneZone(long dcId);
 
     public List<HostVO> listAllHostsInOneZoneByType(Host.Type type, long dcId);

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -18,6 +18,7 @@ package com.cloud.storage;
 
 import java.math.BigDecimal;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
@@ -96,14 +97,6 @@ public interface StorageManager extends StorageService {
             true,
             ConfigKey.Scope.Global,
             null);
-    ConfigKey<Integer> ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class,
-            "convert.vmware.instance.to.kvm.timeout",
-            "Storage",
-            "8",
-            "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host",
-            true,
-            ConfigKey.Scope.Global,
-            null);
     ConfigKey<Boolean> KvmAutoConvergence = new ConfigKey<>(Boolean.class,
             "kvm.auto.convergence",
             "Storage",
@@ -348,6 +341,10 @@ public interface StorageManager extends StorageService {
 
     boolean registerHostListener(String providerUuid, HypervisorHostListener listener);
 
+    Pair<Map<String, String>, Boolean> getStoragePoolNFSMountOpts(StoragePool pool, Map<String, String> details);
+
+    String getStoragePoolMountFailureReason(String error);
+
     boolean connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException;
 
     void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException;

engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java

@@ -264,11 +264,13 @@ public class VirtualMachineProfileImpl implements VirtualMachineProfile {
         _offering = offering;
     }
 
+    @Override
     public void setCpuOvercommitRatio(Float cpuOvercommitRatio) {
         this.cpuOvercommitRatio = cpuOvercommitRatio;
 
     }
 
+    @Override
     public void setMemoryOvercommitRatio(Float memoryOvercommitRatio) {
         this.memoryOvercommitRatio = memoryOvercommitRatio;
 

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java

@@ -45,6 +45,8 @@ import com.cloud.agent.api.CheckOnHostCommand;
 import com.cloud.agent.api.CheckVirtualMachineCommand;
 import com.cloud.agent.api.CleanupNetworkRulesCmd;
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.CreateStoragePoolCommand;
+import com.cloud.agent.api.DeleteStoragePoolCommand;
 import com.cloud.agent.api.MaintainCommand;
 import com.cloud.agent.api.MigrateCommand;
 import com.cloud.agent.api.ModifySshKeysCommand;
@@ -122,8 +124,9 @@ public abstract class AgentAttache {
         StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(),
         ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(),
         CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(),
-        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(),
+        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(),
-            CleanupPersistentNetworkResourceCommand.class.toString()};
+        CreateStoragePoolCommand.class.toString(), DeleteStoragePoolCommand.class.toString(), ModifyStoragePoolCommand.class.toString(),
+        SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), CleanupPersistentNetworkResourceCommand.class.toString()};
     protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() };
     static {
         Arrays.sort(s_commandsAllowedInMaintenanceMode);

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1230,21 +1230,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
                 long destHostId = dest.getHost().getId();
                 vm.setPodIdToDeployIn(dest.getPod().getId());
-                final Long cluster_id = dest.getCluster().getId();
+                final Long clusterId = dest.getCluster().getId();
-                final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
+                updateOverCommitRatioForVmProfile(vmProfile, clusterId);
-                final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
 
-                if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null &&
-                        (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) {
-                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true);
-                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true);
-                } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) {
-                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true);
-                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true);
-                }
-
-                vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue()));
-                vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue()));
                 StartAnswer startAnswer = null;
 
                 try {
@@ -1259,7 +1247,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                     resetVmNicsDeviceId(vm.getId());
                     _networkMgr.prepare(vmProfile, dest, ctx);
                     if (vm.getHypervisorType() != HypervisorType.BareMetal) {
-                        checkAndAttemptMigrateVmAcrossCluster(vm, cluster_id, dest.getStorageForDisks());
+                        checkAndAttemptMigrateVmAcrossCluster(vm, clusterId, dest.getStorageForDisks());
                         volumeMgr.prepare(vmProfile, dest);
                     }
 
@@ -1504,6 +1492,27 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         networkToNetworkNameMap.put(networkVO.getId(), networkName);
     }
 
+    private void updateOverCommitRatioForVmProfile(VirtualMachineProfile vmProfile, long clusterId) {
+        final ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
+        final ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
+        final float parsedClusterCpuDetailCpu = Float.parseFloat(clusterDetailCpu.getValue());
+        final float parsedClusterDetailRam = Float.parseFloat(clusterDetailRam.getValue());
+        UserVmDetailVO vmDetailCpu = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
+        UserVmDetailVO vmDetailRam = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
+
+        if ((vmDetailCpu == null && parsedClusterCpuDetailCpu > 1f) ||
+                (vmDetailCpu != null && Float.parseFloat(vmDetailCpu.getValue()) != parsedClusterCpuDetailCpu)) {
+            userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterDetailCpu.getValue(), true);
+        }
+        if ((vmDetailRam == null && parsedClusterDetailRam > 1f) ||
+                (vmDetailRam != null && Float.parseFloat(vmDetailRam.getValue()) != parsedClusterDetailRam)) {
+            userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterDetailRam.getValue(), true);
+        }
+
+        vmProfile.setCpuOvercommitRatio(Float.parseFloat(clusterDetailCpu.getValue()));
+        vmProfile.setMemoryOvercommitRatio(Float.parseFloat(clusterDetailRam.getValue()));
+    }
+
     /**
      * Setting pod id to null can result in migration of Volumes across pods. This is not desirable for VMs which
      * have a volume in Ready state (happens when a VM is shutdown and started again).
@@ -2001,20 +2010,24 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     }
 
     private void updatePersistenceMap(Map<String, Boolean> vlanToPersistenceMap, NetworkVO networkVO) {
+        if (networkVO == null) {
+            return;
+        }
         NetworkOfferingVO offeringVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId());
-        if (offeringVO != null) {
+        if (offeringVO == null) {
+            return;
+        }
         Pair<String, Boolean> data = getVMNetworkDetails(networkVO, offeringVO.isPersistent());
         Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null;
         if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) {
             vlanToPersistenceMap.put(data.first(), data.second());
         }
     }
-    }
 
     private Map<String, Boolean> getVlanToPersistenceMapForVM(long vmId) {
         List<UserVmJoinVO> userVmJoinVOs = userVmJoinDao.searchByIds(vmId);
         Map<String, Boolean> vlanToPersistenceMap = new HashMap<>();
-        if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) {
+        if (CollectionUtils.isNotEmpty(userVmJoinVOs)) {
             for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) {
                 NetworkVO networkVO = _networkDao.findById(userVmJoinVO.getNetworkId());
                 updatePersistenceMap(vlanToPersistenceMap, networkVO);
@@ -2728,6 +2741,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         _networkMgr.prepareNicForMigration(profile, dest);
         volumeMgr.prepareForMigration(profile, dest);
         profile.setConfigDriveLabel(VmConfigDriveLabel.value());
+        updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId());
 
         final VirtualMachineTO to = toVmTO(profile);
         final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to);
@@ -4777,6 +4791,18 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
     }
 
+    private ApiCommandResourceType getApiCommandResourceTypeForVm(VirtualMachine vm) {
+        switch (vm.getType()) {
+            case DomainRouter:
+                return ApiCommandResourceType.DomainRouter;
+            case ConsoleProxy:
+                return ApiCommandResourceType.ConsoleProxy;
+            case SecondaryStorageVm:
+                return ApiCommandResourceType.SystemVm;
+        }
+        return ApiCommandResourceType.VirtualMachine;
+    }
+
     private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) {
         Host host = _hostDao.findById(vm.getHostId());
         Host poweredHost = _hostDao.findById(vm.getPowerHostId());
@@ -4824,7 +4850,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                     + " -> Running) from out-of-context transition. VM network environment may need to be reset");
 
             ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, vm.getDomainId(),
-                    EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), ApiCommandResourceType.VirtualMachine.toString());
+                EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), getApiCommandResourceTypeForVm(vm).toString());
             logger.info("VM {} is sync-ed to at Running state according to power-on report from hypervisor.", vm.getInstanceName());
             break;
 
@@ -4857,7 +4883,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         case Running:
         case Stopped:
             ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,vm.getDomainId(),
-                    EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), ApiCommandResourceType.VirtualMachine.toString());
+                    EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), getApiCommandResourceTypeForVm(vm).toString());
         case Migrating:
             logger.info("VM {} is at {} and we received a {} report while there is no pending jobs on it"
                             , vm.getInstanceName(), vm.getState(), vm.getPowerState());

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -765,6 +765,14 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                     continue;
                 }
 
+                // Ensure cidr size is equal to 64 for
+                //      - networks other than shared networks
+                //      - shared networks with SLAAC V6 only
+                if (predefined != null && StringUtils.isNotBlank(predefined.getIp6Cidr()) &&
+                        (!GuestType.Shared.equals(offering.getGuestType()) || guru.isSlaacV6Only())) {
+                    _networkModel.checkIp6CidrSizeEqualTo64(predefined.getIp6Cidr());
+                }
+
                 if (network.getId() != -1) {
                     if (network instanceof NetworkVO) {
                         networks.add((NetworkVO) network);
@@ -1033,29 +1041,37 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
     }
 
-    @DB
+    private NicVO persistNicAfterRaceCheck(final NicVO nic, final Long networkId, final NicProfile profile, int deviceId) {
+        return Transaction.execute(new TransactionCallback<NicVO>() {
             @Override
-    public Pair<NicProfile, Integer> allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            public NicVO doInTransaction(TransactionStatus status) {
-            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException {
+                NicVO vo = _nicDao.findByIp4AddressAndNetworkId(profile.getIPv4Address(), networkId);
+                if (vo == null) {
+                    applyProfileToNic(nic, profile, deviceId);
+                    vo = _nicDao.persist(nic);
+                    return vo;
+                } else {
+                    return null;
+                }
+            }
+        });
+    }
 
+    private NicVO checkForRaceAndAllocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException {
         final NetworkVO ntwkVO = _networksDao.findById(network.getId());
         logger.debug("Allocating nic for vm {} in network {} with requested profile {}", vm.getVirtualMachine(), network, requested);
         final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName());
 
-        if (requested != null && requested.getMode() == null) {
+        NicVO vo = null;
-            requested.setMode(network.getMode());
+        boolean retryIpAllocation;
-        }
+        do {
+            retryIpAllocation = false;
             final NicProfile profile = guru.allocate(network, requested, vm);
             if (profile == null) {
                 return null;
             }
 
-        if (isNicAllocatedForNsxPublicNetworkOnVR(network, profile, vm)) {
-            String guruName = "NsxPublicNetworkGuru";
-            NetworkGuru nsxGuru = AdapterBase.getAdapterByName(networkGurus, guruName);
-            nsxGuru.allocate(network, profile, vm);
-        }
-
             if (isDefaultNic != null) {
                 profile.setDefaultNic(isDefaultNic);
             }
@@ -1066,15 +1082,43 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 profile.setMode(network.getMode());
             }
 
-        NicVO vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType());
+            vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType());
 
             DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId());
             if (dcVo.getNetworkType() == NetworkType.Basic) {
                 configureNicProfileBasedOnRequestedIp(requested, profile, network);
             }
 
-        deviceId = applyProfileToNic(vo, profile, deviceId);
+            if (profile.getIpv4AllocationRaceCheck()) {
+                vo = persistNicAfterRaceCheck(vo, network.getId(), profile, deviceId);
+            } else {
+                applyProfileToNic(vo, profile, deviceId);
                 vo = _nicDao.persist(vo);
+            }
+
+            if (vo == null) {
+                if (requested.getRequestedIPv4() != null) {
+                    throw new InsufficientVirtualNetworkCapacityException("Unable to acquire requested Guest IP address " + requested.getRequestedIPv4() + " for network " + network, DataCenter.class, dcVo.getId());
+                } else {
+                    requested.setIPv4Address(null);
+                }
+                retryIpAllocation = true;
+            }
+        } while (retryIpAllocation);
+
+        return vo;
+    }
+
+    @DB
+    @Override
+    public Pair<NicProfile, Integer> allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException {
+
+        if (requested != null && requested.getMode() == null) {
+            requested.setMode(network.getMode());
+        }
+
+        NicVO vo = checkForRaceAndAllocateNic(requested, network, isDefaultNic, deviceId, vm);
 
         final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
         final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
@@ -2723,8 +2767,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
         }
 
-        if (ipv6 && NetUtils.getIp6CidrSize(ip6Cidr) != 64) {
+        if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) {
-            throw new InvalidParameterValueException("IPv6 subnet should be exactly 64-bits in size");
+            _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr);
         }
 
         //TODO(VXLAN): Support VNI specified
@@ -4605,10 +4649,16 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         final NicVO vo = Transaction.execute(new TransactionCallback<NicVO>() {
             @Override
             public NicVO doInTransaction(TransactionStatus status) {
-                NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddress);
+                if (StringUtils.isBlank(macAddress)) {
-                String macAddressToPersist = macAddress;
+                    throw new CloudRuntimeException("Mac address not specified");
+                }
+                String macAddressToPersist = macAddress.trim();
+                if (!NetUtils.isValidMac(macAddressToPersist)) {
+                    throw new CloudRuntimeException("Invalid mac address: " + macAddressToPersist);
+                }
+                NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddressToPersist);
                 if (existingNic != null) {
-                    macAddressToPersist = generateNewMacAddressIfForced(network, macAddress, forced);
+                    macAddressToPersist = generateNewMacAddressIfForced(network, macAddressToPersist, forced);
                 }
                 NicVO vo = new NicVO(network.getGuruName(), vm.getId(), network.getId(), vm.getType());
                 vo.setMacAddress(macAddressToPersist);
@@ -4653,7 +4703,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
                 _networkModel.getNetworkTag(vm.getHypervisorType(), network));
 
-        return new Pair<NicProfile, Integer>(vmNic, Integer.valueOf(deviceId));
+        return new Pair<>(vmNic, Integer.valueOf(deviceId));
     }
 
     protected String getSelectedIpForNicImport(Network network, DataCenter dataCenter, Network.IpAddresses ipAddresses) {
@@ -4697,7 +4747,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     private String generateNewMacAddressIfForced(Network network, String macAddress, boolean forced) {
         if (!forced) {
-            throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() +
+            throw new CloudRuntimeException("NIC with MAC address " + macAddress + " exists on network with ID " + network.getUuid() +
                     " and forced flag is disabled");
         }
         try {

engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.engine.orchestration;
 
 import static org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService.NetworkLockTimeout;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -31,6 +32,7 @@ import java.util.List;
 import java.util.Map;
 
 import com.cloud.dc.DataCenter;
+import com.cloud.exception.InsufficientVirtualNetworkCapacityException;
 import com.cloud.network.IpAddressManager;
 import com.cloud.utils.Pair;
 import org.junit.Assert;
@@ -39,6 +41,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentMatchers;
+import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 
 import com.cloud.api.query.dao.DomainRouterJoinDao;
@@ -71,6 +74,8 @@ import com.cloud.network.vpc.VpcManager;
 import com.cloud.network.vpc.VpcVO;
 import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallback;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.Ip;
 import com.cloud.vm.DomainRouterVO;
@@ -890,4 +895,118 @@ public class NetworkOrchestratorTest extends TestCase {
         verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value());
         verify(testOrchestrator._networksDao, times(1)).releaseFromLockTable(networkId);
     }
+
+    @Test(expected = InsufficientVirtualNetworkCapacityException.class)
+    public void testImportNicAcquireGuestIPFailed() throws Exception {
+        DataCenter dataCenter = Mockito.mock(DataCenter.class);
+        VirtualMachine vm = mock(VirtualMachine.class);
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated);
+        Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId);
+        long dataCenterId = 1L;
+        Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId);
+        Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class);
+        String ipAddress = "10.1.10.10";
+        Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress);
+        Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null);
+        Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp));
+        String macAddress = "02:01:01:82:00:01";
+        int deviceId = 0;
+        testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false);
+    }
+
+    @Test(expected = InsufficientVirtualNetworkCapacityException.class)
+    public void testImportNicAutoAcquireGuestIPFailed() throws Exception {
+        DataCenter dataCenter = Mockito.mock(DataCenter.class);
+        VirtualMachine vm = mock(VirtualMachine.class);
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated);
+        Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId);
+        long dataCenterId = 1L;
+        Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId);
+        Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class);
+        String ipAddress = "auto";
+        Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress);
+        Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null);
+        Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp));
+        String macAddress = "02:01:01:82:00:01";
+        int deviceId = 0;
+        testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false);
+    }
+
+    @Test
+    public void testImportNicNoIP4Address() throws Exception {
+        DataCenter dataCenter = Mockito.mock(DataCenter.class);
+        Long vmId = 1L;
+        Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM;
+        VirtualMachine vm = mock(VirtualMachine.class);
+        Mockito.when(vm.getId()).thenReturn(vmId);
+        Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType);
+        Long networkId = 1L;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getId()).thenReturn(networkId);
+        Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class);
+        Mockito.when(ipAddresses.getIp4Address()).thenReturn(null);
+        URI broadcastUri = URI.create("vlan://123");
+        NicVO nic = mock(NicVO.class);
+        Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri);
+        String macAddress = "02:01:01:82:00:01";
+        int deviceId = 1;
+        Integer networkRate = 200;
+        Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate);
+        Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false);
+        Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag");
+        try (MockedStatic<Transaction> transactionMocked = Mockito.mockStatic(Transaction.class)) {
+            transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic);
+            Pair<NicProfile, Integer> nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false);
+            verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId);
+            verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network);
+            verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network);
+            assertEquals(deviceId, nicProfileIntegerPair.second().intValue());
+            NicProfile nicProfile = nicProfileIntegerPair.first();
+            assertEquals(broadcastUri, nicProfile.getBroadCastUri());
+            assertEquals(networkRate, nicProfile.getNetworkRate());
+            assertFalse(nicProfile.isSecurityGroupEnabled());
+            assertEquals("testtag", nicProfile.getName());
+        }
+    }
+
+    @Test
+    public void testImportNicWithIP4Address() throws Exception {
+        DataCenter dataCenter = Mockito.mock(DataCenter.class);
+        Long vmId = 1L;
+        Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM;
+        VirtualMachine vm = mock(VirtualMachine.class);
+        Mockito.when(vm.getId()).thenReturn(vmId);
+        Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType);
+        Long networkId = 1L;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getId()).thenReturn(networkId);
+        String ipAddress = "10.1.10.10";
+        Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class);
+        Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress);
+        URI broadcastUri = URI.create("vlan://123");
+        NicVO nic = mock(NicVO.class);
+        Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri);
+        String macAddress = "02:01:01:82:00:01";
+        int deviceId = 1;
+        Integer networkRate = 200;
+        Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate);
+        Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false);
+        Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag");
+        try (MockedStatic<Transaction> transactionMocked = Mockito.mockStatic(Transaction.class)) {
+            transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic);
+            Pair<NicProfile, Integer> nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false);
+            verify(testOrchestrator, times(1)).getSelectedIpForNicImport(network, dataCenter, ipAddresses);
+            verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId);
+            verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network);
+            verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network);
+            assertEquals(deviceId, nicProfileIntegerPair.second().intValue());
+            NicProfile nicProfile = nicProfileIntegerPair.first();
+            assertEquals(broadcastUri, nicProfile.getBroadCastUri());
+            assertEquals(networkRate, nicProfile.getNetworkRate());
+            assertFalse(nicProfile.isSecurityGroupEnabled());
+            assertEquals("testtag", nicProfile.getName());
+        }
+    }
 }

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -141,6 +141,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listByHostCapability(Host.Type type, Long clusterId, Long podId, long dcId, String hostCapabilty);
 
+    List<HostVO> listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty);
+
     List<HostVO> listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType);
 
     HostVO findByName(String name);

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -341,6 +341,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         ClusterHypervisorSearch.and("hypervisor", ClusterHypervisorSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ);
         ClusterHypervisorSearch.and("type", ClusterHypervisorSearch.entity().getType(), SearchCriteria.Op.EQ);
         ClusterHypervisorSearch.and("status", ClusterHypervisorSearch.entity().getStatus(), SearchCriteria.Op.EQ);
+        ClusterHypervisorSearch.and("resourceState", ClusterHypervisorSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
         ClusterHypervisorSearch.done();
 
         UnmanagedDirectConnectSearch = createSearchBuilder();
@@ -1506,12 +1507,42 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    @Override
+    public List<HostVO> listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty) {
+        SearchBuilder<DetailVO> hostCapabilitySearch = _detailsDao.createSearchBuilder();
+        DetailVO tagEntity = hostCapabilitySearch.entity();
+        hostCapabilitySearch.and("capability", tagEntity.getName(), SearchCriteria.Op.EQ);
+        hostCapabilitySearch.and("value", tagEntity.getValue(), SearchCriteria.Op.EQ);
+
+        SearchBuilder<HostVO> hostSearch = createSearchBuilder();
+        HostVO entity = hostSearch.entity();
+        hostSearch.and("clusterId", entity.getClusterId(), SearchCriteria.Op.EQ);
+        hostSearch.and("hypervisor", entity.getHypervisorType(), SearchCriteria.Op.EQ);
+        hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ);
+        hostSearch.and("status", entity.getStatus(), SearchCriteria.Op.EQ);
+        hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ);
+        hostSearch.join("hostCapabilitySearch", hostCapabilitySearch, entity.getId(), tagEntity.getHostId(), JoinBuilder.JoinType.INNER);
+
+        SearchCriteria<HostVO> sc = hostSearch.create();
+        sc.setJoinParameters("hostCapabilitySearch", "value", Boolean.toString(true));
+        sc.setJoinParameters("hostCapabilitySearch", "capability", hostCapabilty);
+
+        sc.setParameters("clusterId", clusterId);
+        sc.setParameters("hypervisor", hypervisorType);
+        sc.setParameters("type", Type.Routing);
+        sc.setParameters("status", Status.Up);
+        sc.setParameters("resourceState", ResourceState.Enabled);
+        return listBy(sc);
+    }
+
+    @Override
     public List<HostVO> listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType) {
         SearchCriteria<HostVO> sc = ClusterHypervisorSearch.create();
         sc.setParameters("clusterId", clusterId);
         sc.setParameters("hypervisor", hypervisorType);
         sc.setParameters("type", Type.Routing);
         sc.setParameters("status", Status.Up);
+        sc.setParameters("resourceState", ResourceState.Enabled);
         return listBy(sc);
     }
 

engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java

@@ -18,9 +18,12 @@
  */
 package com.cloud.storage.dao;
 
+import java.util.List;
+
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
 
 import com.cloud.utils.db.GenericDao;
 
 public interface SnapshotDetailsDao extends GenericDao<SnapshotDetailsVO, Long>, ResourceDetailsDao<SnapshotDetailsVO> {
+    public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String key);
 }

engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java

@@ -18,11 +18,44 @@
  */
 package com.cloud.storage.dao;
 
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
 
+import com.cloud.utils.db.TransactionLegacy;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 public class SnapshotDetailsDaoImpl extends ResourceDetailsDaoBase<SnapshotDetailsVO> implements SnapshotDetailsDao {
+    private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE ss.data_center_id = ? AND s.name = ?";
+
     @Override
     public void addDetail(long resourceId, String key, String value, boolean display) {
         super.addDetail(new SnapshotDetailsVO(resourceId, key, value, display));
     }
+
+    public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String key) {
+        StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE);
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        List<SnapshotDetailsVO> snapshotDetailsOnZone = new ArrayList<SnapshotDetailsVO>();
+        try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) {
+            if (pstmt != null) {
+                pstmt.setLong(1, dcId);
+                pstmt.setString(2, key);
+                try (ResultSet rs = pstmt.executeQuery();) {
+                    while (rs.next()) {
+                        snapshotDetailsOnZone.add(toEntityBean(rs, false));
+                    }
+                } catch (SQLException e) {
+                    throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e);
+                }
+            }
+            return snapshotDetailsOnZone;
+        } catch (SQLException e) {
+            throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e);
+        }
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java

@@ -160,4 +160,6 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
     List<VolumeVO> listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(long accountId, List<Long> diskOfferingIds, List<Long> vmIds);
 
     List<VolumeVO> searchRemovedByVms(List<Long> vmIds, Long batchSize);
+
+    VolumeVO findOneByIScsiName(String iScsiName);
 }

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java

@@ -395,6 +395,7 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
         AllFieldsSearch.and("updatedCount", AllFieldsSearch.entity().getUpdatedCount(), Op.EQ);
         AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), Op.EQ);
         AllFieldsSearch.and("passphraseId", AllFieldsSearch.entity().getPassphraseId(), Op.EQ);
+        AllFieldsSearch.and("iScsiName", AllFieldsSearch.entity().get_iScsiName(), Op.EQ);
         AllFieldsSearch.done();
 
         RootDiskStateSearch = createSearchBuilder();
@@ -910,4 +911,10 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
         Filter filter = new Filter(VolumeVO.class, "id", true, 0L, batchSize);
         return searchIncludingRemoved(sc, filter, null, false);
     }
+
+    public VolumeVO findOneByIScsiName(String iScsiName) {
+        SearchCriteria<VolumeVO> sc = AllFieldsSearch.create();
+        sc.setParameters("iScsiName", iScsiName);
+        return findOneIncludingRemovedBy(sc);
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java

@@ -75,8 +75,10 @@ public interface VolumeStatsDao extends GenericDao<VolumeStatsVO, Long> {
     /**
      * Removes (expunges) all Volume stats with {@code timestamp} less than
      * a given Date.
-     * @param limit the maximum date to keep stored. Records that exceed this limit will be removed.
+     * @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed.
+     * @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query.
+     *                      If 0 or negative, no limit is used.
      */
-    void removeAllByTimestampLessThan(Date limit);
+    void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery);
 
 }

engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java

@@ -21,6 +21,8 @@ import java.util.List;
 
 import javax.annotation.PostConstruct;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.cloud.utils.db.Filter;
@@ -33,6 +35,8 @@ import com.cloud.storage.VolumeStatsVO;
 @Component
 public class VolumeStatsDaoImpl extends GenericDaoBase<VolumeStatsVO, Long> implements VolumeStatsDao {
 
+    protected Logger logger = LogManager.getLogger(getClass());
+
     protected SearchBuilder<VolumeStatsVO> volumeIdSearch;
     protected SearchBuilder<VolumeStatsVO> volumeIdTimestampGreaterThanEqualSearch;
     protected SearchBuilder<VolumeStatsVO> volumeIdTimestampLessThanEqualSearch;
@@ -116,9 +120,21 @@ public class VolumeStatsDaoImpl extends GenericDaoBase<VolumeStatsVO, Long> impl
     }
 
     @Override
-    public void removeAllByTimestampLessThan(Date limit) {
+    public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
         SearchCriteria<VolumeStatsVO> sc = timestampSearch.create();
-        sc.setParameters(TIMESTAMP, limit);
+        sc.setParameters(TIMESTAMP, limitDate);
-        expunge(sc);
+
+        logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate));
+
+        long totalRemoved = 0;
+        long removed;
+
+        do {
+            removed = expunge(sc, limitPerQuery);
+            totalRemoved += removed;
+            logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
+        } while (limitPerQuery > 0 && removed >= limitPerQuery);
+
+        logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate));
     }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -127,6 +127,10 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     List<StoragePoolVO> findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType, String keyword);
 
+    List<StoragePoolVO> findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType);
+
+    List<StoragePoolVO> findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType);
+
     List<StoragePoolVO> findLocalStoragePoolsByHostAndTags(long hostId, String[] tags);
 
     List<StoragePoolVO> listLocalStoragePoolByPath(long datacenterId, String path);
@@ -141,6 +145,8 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     List<StoragePoolVO> findPoolsByStorageType(Storage.StoragePoolType storageType);
 
+    StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath);
+
     List<StoragePoolVO> listStoragePoolsWithActiveVolumesByOfferingId(long offeringid);
 
     Pair<List<Long>, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId,

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.storage.Storage;
 import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
 import org.apache.commons.collections.CollectionUtils;
@@ -35,7 +36,6 @@ import org.apache.commons.collections.CollectionUtils;
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.ScopeType;
-import com.cloud.storage.Storage;
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StoragePoolStatus;
 import com.cloud.storage.StoragePoolTagVO;
@@ -622,6 +622,28 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         return sc.list();
     }
 
+    @Override
+    public List<StoragePoolVO> findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) {
+        QueryBuilder<StoragePoolVO> sc = QueryBuilder.create(StoragePoolVO.class);
+        sc.and(sc.entity().getDataCenterId(), Op.EQ, dataCenterId);
+        sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up);
+        sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE);
+        sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType);
+        sc.and(sc.entity().getPoolType(), Op.EQ, poolType);
+        return sc.list();
+    }
+
+    @Override
+    public List<StoragePoolVO> findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) {
+        QueryBuilder<StoragePoolVO> sc = QueryBuilder.create(StoragePoolVO.class);
+        sc.and(sc.entity().getClusterId(), Op.EQ, clusterId);
+        sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up);
+        sc.and(sc.entity().getScope(), Op.EQ, ScopeType.CLUSTER);
+        sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType);
+        sc.and(sc.entity().getPoolType(), Op.EQ, poolType);
+        return sc.list();
+    }
+
     @Override
     public void deletePoolTags(long poolId) {
         _tagsDao.deleteTags(poolId);
@@ -660,6 +682,16 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         return listBy(sc);
     }
 
+    @Override
+    public StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath) {
+        SearchCriteria<StoragePoolVO> sc = AllFieldSearch.create();
+        sc.setParameters("datacenterId", zoneId);
+        if (datastorePath != null) {
+            sc.addAnd("path", Op.LIKE,  "%/" + datastorePath);
+        }
+        return findOneBy(sc);
+    }
+
     @Override
     public List<StoragePoolVO> listStoragePoolsWithActiveVolumesByOfferingId(long offeringId) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java

@@ -100,6 +100,9 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
     @Override
     public TemplateInfo getTemplate(long templateId, DataStore store) {
         VMTemplateVO templ = imageDataDao.findById(templateId);
+        if (templ == null) {
+            return null;
+        }
         if (store == null && !templ.isDirectDownload()) {
             TemplateObject tmpl = TemplateObject.getTemplate(templ, null, null);
             return tmpl;

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -82,6 +82,10 @@ public class TemplateObject implements TemplateInfo {
     }
 
     protected void configure(VMTemplateVO template, DataStore dataStore) {
+        if (template == null) {
+            String msg = String.format("Template Object is not properly initialised %s", this.toString());
+            logger.warn(msg);
+        }
         imageVO = template;
         this.dataStore = dataStore;
     }
@@ -98,6 +102,10 @@ public class TemplateObject implements TemplateInfo {
     }
 
     public VMTemplateVO getImage() {
+        if (imageVO == null) {
+            String msg = String.format("Template Object is not properly initialised %s", this.toString());
+            logger.error(msg);
+        } // somehow the nullpointer is needed : refacter needed!?!
         return imageVO;
     }
 

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -42,7 +42,9 @@ import com.cloud.storage.StoragePool;
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StorageService;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -54,7 +56,9 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
 import javax.inject.Inject;
+
 import java.util.List;
+import java.util.Map;
 
 public class DefaultHostListener implements HypervisorHostListener {
     protected Logger logger = LogManager.getLogger(getClass());
@@ -126,7 +130,9 @@ public class DefaultHostListener implements HypervisorHostListener {
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
-        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool);
+        Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
+
+        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
         cmd.setWait(modifyStoragePoolCommandWait);
         logger.debug(String.format("Sending modify storage pool command to agent: %d for storage pool: %d with timeout %d seconds",
                 hostId, poolId, cmd.getWait()));
@@ -139,7 +145,7 @@ public class DefaultHostListener implements HypervisorHostListener {
         if (!answer.getResult()) {
             String msg = "Unable to attach storage pool" + poolId + " to the host" + hostId;
             alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg);
-            throw new CloudRuntimeException("Unable establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() +
+            throw new CloudRuntimeException("Unable to establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() +
                 pool.getId());
         }
 

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -505,9 +505,11 @@ public class VolumeServiceImpl implements VolumeService {
                             _snapshotStoreDao.remove(snapStoreVo.getId());
                         }
                     } else {
+                        if (!StoragePoolType.StorPool.equals(storagePoolVO.getPoolType())) {
                             _snapshotStoreDao.remove(snapStoreVo.getId());
                         }
                     }
+                }
                 snapshotApiService.markVolumeSnapshotsAsDestroyed(vo);
             } else {
                 vo.processEvent(Event.OperationFailed);

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java

@@ -346,7 +346,7 @@ public class VeeamClient {
                         String type = pair.second();
                         String path = url.replace(apiURI.toString(), "");
                         if (type.equals("RestoreSession")) {
-                            return checkIfRestoreSessionFinished(type, path);
+                            checkIfRestoreSessionFinished(type, path);
                         }
                     }
                     return true;
@@ -362,17 +362,29 @@ public class VeeamClient {
         return false;
     }
 
-    protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException {
+
-        for (int j = 0; j < this.restoreTimeout; j++) {
+    /**
+     * Checks the status of the restore session. Checked states are "Success" and "Failure".<br/>
+     * There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,<br/>
+     * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}.
+     */
+    protected void checkIfRestoreSessionFinished(String type, String path) throws IOException {
+        for (int j = 0; j < restoreTimeout; j++) {
             HttpResponse relatedResponse = get(path);
             RestoreSession session = parseRestoreSessionResponse(relatedResponse);
             if (session.getResult().equals("Success")) {
-                return true;
+                return;
             }
+
             if (session.getResult().equalsIgnoreCase("Failed")) {
                 String sessionUid = session.getUid();
+                LOG.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].",
+                        sessionUid, session.getVmDisplayName(),
+                        getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":"))));
                 throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid));
             }
+            LOG.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout));
+
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException ignored) {
@@ -931,6 +943,29 @@ public class VeeamClient {
         return new Pair<>(result.first(), restoreLocation);
     }
 
+    /**
+     * Tries to retrieve the error's description of the Veeam restore task that resulted in an error.
+     * @param uid Session uid in Veeam of the restore process;
+     * @return the description found in Veeam about the cause of error in the restore process.
+     */
+    protected String getRestoreVmErrorDescription(String uid) {
+        LOG.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid));
+        List<String> cmds = Arrays.asList(
+                String.format("$restoreUid = '%s'", uid),
+                "$restore = Get-VBRRestoreSession -Id $restoreUid",
+                "if ($restore) {",
+                    "Write-Output $restore.Description",
+                "} else {",
+                    "Write-Output 'Cannot find restore session with provided uid $restoreUid'",
+                "}"
+        );
+        Pair<Boolean, String> result = executePowerShellCommands(cmds);
+        if (result != null && result.first()) {
+            return result.second();
+        }
+        return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid);
+    }
+
     private boolean isLegacyServer() {
         return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11);
     }

plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java

@@ -59,6 +59,8 @@ public class VeeamClientTest {
     private VeeamClient mockClient;
     private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
+    private VeeamClient mock = Mockito.mock(VeeamClient.class);
+
     @Rule
     public WireMockRule wireMockRule = new WireMockRule(9399);
 
@@ -163,7 +165,7 @@ public class VeeamClientTest {
             Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
             Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession);
             Mockito.when(restoreSession.getResult()).thenReturn("No Success");
-            Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod();
+            Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"));
             mockClient.checkIfRestoreSessionFinished("RestoreTest", "any");
             fail();
         } catch (Exception e) {
@@ -172,6 +174,42 @@ public class VeeamClientTest {
         Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
     }
 
+    @Test
+    public void getRestoreVmErrorDescriptionTestFindErrorDescription() {
+        Pair<Boolean, String> response = new Pair<>(true, "Example of error description found in Veeam.");
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Example of error description found in Veeam.", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() {
+        Pair<Boolean, String> response = new Pair<>(true, "Cannot find restore session with provided uid uuid");
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Cannot find restore session with provided uid uuid", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() {
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() {
+        Pair<Boolean, String> response = new Pair<>(false, null);
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
+    }
+
+
     private void verifyBackupMetrics(Map<String, Backup.Metric> metrics) {
         Assert.assertEquals(2, metrics.size());
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.hypervisor.kvm.resource;
 
+import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION;
 import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION;
 
 import java.io.BufferedReader;
@@ -306,6 +307,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     public static final String TUNGSTEN_PATH = "scripts/vm/network/tungsten";
 
+    public static final String INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD = "virt-v2v --version";
+    // virt-v2v --version => sample output: virt-v2v 1.42.0rhel=8,release=22.module+el8.10.0+1590+a67ab969
+    public static final String OVF_EXPORT_SUPPORTED_CHECK_CMD = "ovftool --version";
+    // ovftool --version => sample output: VMware ovftool 4.6.0 (build-21452615)
+    public static final String OVF_EXPORT_TOOl_GET_VERSION_CMD = "ovftool --version | awk '{print $3}'";
+
+    public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win";
+    public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win";
+    public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit";
+
     private String modifyVlanPath;
     private String versionStringPath;
     private String patchScriptPath;
@@ -3647,6 +3658,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         cmd.setIqn(getIqn());
         cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption()));
         cmd.setHostTags(getHostTags());
+        cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion()));
         HealthCheckResult healthCheckResult = getHostHealthCheckResult();
         if (healthCheckResult != HealthCheckResult.IGNORE) {
             cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS);
@@ -5163,6 +5175,48 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return false;
     }
 
+    public boolean hostSupportsInstanceConversion() {
+        int exitValue = Script.runSimpleBashScriptForExitValue(INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD);
+        if (isUbuntuHost() && exitValue == 0) {
+            exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_NBDKIT_PKG_CHECK_CMD);
+        }
+        return exitValue == 0;
+    }
+
+    public boolean hostSupportsWindowsGuestConversion() {
+        if (isUbuntuHost()) {
+            int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD);
+            return exitValue == 0;
+        }
+        int exitValue = Script.runSimpleBashScriptForExitValue(WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD);
+        return exitValue == 0;
+    }
+
+    public boolean hostSupportsOvfExport() {
+        int exitValue = Script.runSimpleBashScriptForExitValue(OVF_EXPORT_SUPPORTED_CHECK_CMD);
+        return exitValue == 0;
+    }
+
+    public boolean ovfExportToolSupportsParallelThreads() {
+        String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD);
+        if (StringUtils.isBlank(ovfExportToolVersion)) {
+            return false;
+        }
+        String[] ovfExportToolVersions = ovfExportToolVersion.trim().split("\\.");
+        if (ovfExportToolVersions.length > 1) {
+            try {
+                int majorVersion = Integer.parseInt(ovfExportToolVersions[0]);
+                int minorVersion = Integer.parseInt(ovfExportToolVersions[1]);
+                //ovftool version >= 4.4 supports parallel threads
+                if (majorVersion > 4 || (majorVersion == 4 && minorVersion >= 4)) {
+                    return true;
+                }
+            } catch (NumberFormatException ignored) {
+            }
+        }
+        return false;
+    }
+
     protected void setCpuTopology(CpuModeDef cmd, int vCpusInDef, Map<String, String> details) {
         if (!enableManuallySettingCpuTopologyOnKvmVm) {
             LOGGER.debug(String.format("Skipping manually setting CPU topology on VM's XML due to it is disabled in agent.properties {\"property\": \"%s\", \"value\": %s}.",

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java

@@ -16,6 +16,12 @@
 // under the License.
 package com.cloud.hypervisor.kvm.resource;
 
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+
 public class LibvirtStoragePoolDef {
     public enum PoolType {
         ISCSI("iscsi"), NETFS("netfs"), loggerICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex");
@@ -55,6 +61,7 @@ public class LibvirtStoragePoolDef {
     private String _authUsername;
     private AuthenticationType _authType;
     private String _secretUuid;
+    private Set<String> _nfsMountOpts = new HashSet<>();
 
     public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, int port, String dir, String targetPath) {
         _poolType = type;
@@ -75,6 +82,15 @@ public class LibvirtStoragePoolDef {
         _targetPath = targetPath;
     }
 
+    public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, String dir, String targetPath, List<String> nfsMountOpts) {
+        this(type, poolName, uuid, host, dir, targetPath);
+        if (CollectionUtils.isNotEmpty(nfsMountOpts)) {
+            for (String nfsMountOpt : nfsMountOpts) {
+                this._nfsMountOpts.add(nfsMountOpt);
+            }
+        }
+    }
+
     public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String sourceHost, int sourcePort, String dir, String authUsername, AuthenticationType authType,
             String secretUuid) {
         _poolType = type;
@@ -124,28 +140,47 @@ public class LibvirtStoragePoolDef {
         return _authType;
     }
 
+    public Set<String> getNfsMountOpts() {
+        return _nfsMountOpts;
+    }
+
     @Override
     public String toString() {
         StringBuilder storagePoolBuilder = new StringBuilder();
-        if (_poolType == PoolType.GLUSTERFS) {
+        String poolTypeXML;
-            /* libvirt mounts a Gluster volume, similar to NFS */
+        switch (_poolType) {
-            storagePoolBuilder.append("<pool type='netfs'>\n");
+            case NETFS:
+                if (_nfsMountOpts != null) {
+                    poolTypeXML = "netfs' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0";
                 } else {
-            storagePoolBuilder.append("<pool type='");
+                    poolTypeXML = _poolType.toString();
-            storagePoolBuilder.append(_poolType);
-            storagePoolBuilder.append("'>\n");
                 }
+                break;
+            case GLUSTERFS:
+                /* libvirt mounts a Gluster volume, similar to NFS */
+                poolTypeXML = "netfs";
+                break;
+            default:
+                poolTypeXML = _poolType.toString();
+        }
+
+        storagePoolBuilder.append("<pool type='");
+        storagePoolBuilder.append(poolTypeXML);
+        storagePoolBuilder.append("'>\n");
 
         storagePoolBuilder.append("<name>" + _poolName + "</name>\n");
         if (_uuid != null)
             storagePoolBuilder.append("<uuid>" + _uuid + "</uuid>\n");
-        if (_poolType == PoolType.NETFS) {
+
+        switch (_poolType) {
+            case NETFS:
                 storagePoolBuilder.append("<source>\n");
                 storagePoolBuilder.append("<host name='" + _sourceHost + "'/>\n");
                 storagePoolBuilder.append("<dir path='" + _sourceDir + "'/>\n");
                 storagePoolBuilder.append("</source>\n");
-        }
+                break;
-        if (_poolType == PoolType.RBD) {
+
+            case RBD:
                 storagePoolBuilder.append("<source>\n");
                 for (String sourceHost : _sourceHost.split(",")) {
                     storagePoolBuilder.append("<host name='");
@@ -164,8 +199,9 @@ public class LibvirtStoragePoolDef {
                     storagePoolBuilder.append("</auth>\n");
                 }
                 storagePoolBuilder.append("</source>\n");
-        }
+                break;
-        if (_poolType == PoolType.GLUSTERFS) {
+
+            case GLUSTERFS:
                 storagePoolBuilder.append("<source>\n");
                 storagePoolBuilder.append("<host name='");
                 storagePoolBuilder.append(_sourceHost);
@@ -181,12 +217,21 @@ public class LibvirtStoragePoolDef {
                 storagePoolBuilder.append(_poolType);
                 storagePoolBuilder.append("'/>\n");
                 storagePoolBuilder.append("</source>\n");
+                break;
         }
+
         if (_poolType != PoolType.RBD && _poolType != PoolType.POWERFLEX) {
             storagePoolBuilder.append("<target>\n");
             storagePoolBuilder.append("<path>" + _targetPath + "</path>\n");
             storagePoolBuilder.append("</target>\n");
         }
+        if (_poolType == PoolType.NETFS && _nfsMountOpts != null) {
+            storagePoolBuilder.append("<fs:mount_opts>\n");
+            for (String options : _nfsMountOpts) {
+                storagePoolBuilder.append("<fs:option name='" + options + "'/>\n");
+            }
+            storagePoolBuilder.append("</fs:mount_opts>\n");
+        }
         storagePoolBuilder.append("</pool>\n");
         return storagePoolBuilder.toString();
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java

@@ -38,6 +38,19 @@ import org.xml.sax.SAXException;
 public class LibvirtStoragePoolXMLParser {
     protected Logger logger = LogManager.getLogger(getClass());
 
+    private List<String> getNFSMountOptsFromRootElement(Element rootElement) {
+        List<String> nfsMountOpts = new ArrayList<>();
+        Element mountOpts = (Element) rootElement.getElementsByTagName("fs:mount_opts").item(0);
+        if (mountOpts != null) {
+            NodeList options = mountOpts.getElementsByTagName("fs:option");
+            for (int i = 0; i < options.getLength(); i++) {
+                Element option = (Element) options.item(i);
+                nfsMountOpts.add(option.getAttribute("name"));
+            }
+        }
+        return nfsMountOpts;
+    }
+
     public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) {
         DocumentBuilder builder;
         try {
@@ -95,12 +108,16 @@ public class LibvirtStoragePoolXMLParser {
                         poolName, uuid, host, port, path, targetPath);
             } else {
                 String path = getAttrValue("dir", "path", source);
-
                 Element target = (Element)rootElement.getElementsByTagName("target").item(0);
                 String targetPath = getTagValue("path", target);
 
+                if (type.equalsIgnoreCase("netfs")) {
+                    List<String> nfsMountOpts = getNFSMountOptsFromRootElement(rootElement);
+                    return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath, nfsMountOpts);
+                } else {
                     return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath);
                 }
+            }
         } catch (ParserConfigurationException e) {
             logger.debug(e.toString());
         } catch (SAXException e) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java

@@ -0,0 +1,57 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CheckConvertInstanceAnswer;
+import com.cloud.agent.api.CheckConvertInstanceCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+
+@ResourceWrapper(handles =  CheckConvertInstanceCommand.class)
+public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper<CheckConvertInstanceCommand, Answer, LibvirtComputingResource> {
+
+    private static final Logger s_logger = Logger.getLogger(LibvirtCheckConvertInstanceCommandWrapper.class);
+
+    @Override
+    public Answer execute(CheckConvertInstanceCommand cmd, LibvirtComputingResource serverResource) {
+        if (!serverResource.hostSupportsInstanceConversion()) {
+            String msg = String.format("Cannot convert the instance from VMware as the virt-v2v binary is not found on host %s. " +
+                    "Please install virt-v2v%s on the host before attempting the instance conversion.", serverResource.getPrivateIp(), serverResource.isUbuntuHost()? ", nbdkit" : "");
+            s_logger.info(msg);
+            return new CheckConvertInstanceAnswer(cmd, false, msg);
+        }
+
+        if (cmd.getCheckWindowsGuestConversionSupport() && !serverResource.hostSupportsWindowsGuestConversion()) {
+            String msg = String.format("Cannot convert the instance from VMware as the virtio-win package is not found on host %s. " +
+                    "Please install virtio-win package on the host before attempting the windows guest instance conversion.", serverResource.getPrivateIp());
+            s_logger.info(msg);
+            return new CheckConvertInstanceAnswer(cmd, false, msg);
+        }
+
+        if (serverResource.hostSupportsOvfExport()) {
+            return new CheckConvertInstanceAnswer(cmd, true, true, "");
+        }
+
+        return new CheckConvertInstanceAnswer(cmd, true, "");
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java

@@ -62,8 +62,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
     private static final List<Hypervisor.HypervisorType> supportedInstanceConvertSourceHypervisors =
             List.of(Hypervisor.HypervisorType.VMware);
 
-    protected static final String checkIfConversionIsSupportedCommand = "which virt-v2v";
-
     @Override
     public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) {
         RemoteInstanceTO sourceInstance = cmd.getSourceInstance();
@@ -74,9 +72,9 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
         DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation();
         long timeout = (long) cmd.getWait() * 1000;
 
-        if (!isInstanceConversionSupportedOnHost()) {
+        if (cmd.getCheckConversionSupport() && !serverResource.hostSupportsInstanceConversion()) {
             String msg = String.format("Cannot convert the instance %s from VMware as the virt-v2v binary is not found. " +
-                    "Please install virt-v2v on the host before attempting the instance conversion", sourceInstanceName);
+                    "Please install virt-v2v%s on the host before attempting the instance conversion.", sourceInstanceName, serverResource.isUbuntuHost()? ", nbdkit" : "");
             logger.info(msg);
             return new ConvertInstanceAnswer(cmd, false, msg);
         }
@@ -94,18 +92,48 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
 
         logger.info(String.format("Attempting to convert the instance %s from %s to KVM",
                 sourceInstanceName, sourceHypervisorType));
-        final String convertInstanceUrl = getConvertInstanceUrl(sourceInstance);
-        final String temporaryConvertUuid = UUID.randomUUID().toString();
-        final String temporaryPasswordFilePath = createTemporaryPasswordFileAndRetrievePath(sourceInstance);
         final String temporaryConvertPath = temporaryStoragePool.getLocalPath();
+
+        String ovfTemplateDirOnConversionLocation;
+        String sourceOVFDirPath;
+        boolean ovfExported = false;
+        if (cmd.getExportOvfToConversionLocation()) {
+            String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance);
+            if (StringUtils.isBlank(exportInstanceOVAUrl)) {
+                String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName);
+                logger.error(err);
+                return new ConvertInstanceAnswer(cmd, false, err);
+            }
+
+            int noOfThreads = cmd.getThreadsCountToExportOvf();
+            if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) {
+                noOfThreads = 0;
+            }
+            ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString();
+            temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation);
+            sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation);
+            ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, timeout);
+            if (!ovfExported) {
+                String err = String.format("Export OVA for the VM %s failed", sourceInstanceName);
+                logger.error(err);
+                return new ConvertInstanceAnswer(cmd, false, err);
+            }
+            sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName);
+        } else {
+            ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation();
+            sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation);
+        }
+
+        logger.info(String.format("Attempting to convert the OVF %s of the instance %s from %s to KVM", ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType));
+        final String temporaryConvertUuid = UUID.randomUUID().toString();
         boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled();
 
         try {
-            boolean result = performInstanceConversion(convertInstanceUrl, sourceInstanceName, temporaryPasswordFilePath,
+            boolean result = performInstanceConversion(sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid,
-                    temporaryConvertPath, temporaryConvertUuid, timeout, verboseModeEnabled);
+                    timeout, verboseModeEnabled);
             if (!result) {
-                String err = String.format("The virt-v2v conversion of the instance %s failed. " +
+                String err = String.format("The virt-v2v conversion for the OVF %s failed. " +
-                                "Please check the agent logs for the virt-v2v output", sourceInstanceName);
+                                "Please check the agent logs for the virt-v2v output", ovfTemplateDirOnConversionLocation);
                 logger.error(err);
                 return new ConvertInstanceAnswer(cmd, false, err);
             }
@@ -130,8 +158,11 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
             logger.error(error, e);
             return new ConvertInstanceAnswer(cmd, false, error);
         } finally {
-            logger.debug("Cleaning up instance conversion temporary password file");
+            if (ovfExported && StringUtils.isNotBlank(ovfTemplateDirOnConversionLocation)) {
-            Script.runSimpleBashScript(String.format("rm -rf %s", temporaryPasswordFilePath));
+                String sourceOVFDir = String.format("%s/%s", temporaryConvertPath, ovfTemplateDirOnConversionLocation);
+                logger.debug("Cleaning up exported OVA at dir " + sourceOVFDir);
+                Script.runSimpleBashScript("rm -rf " + sourceOVFDir);
+            }
             if (conversionTemporaryLocation instanceof NfsTO) {
                 logger.debug("Cleaning up secondary storage temporary location");
                 storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid());
@@ -155,6 +186,27 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
                 supportedInstanceConvertSourceHypervisors.contains(sourceHypervisorType);
     }
 
+    private String getExportInstanceOVAUrl(RemoteInstanceTO sourceInstance) {
+        String url = null;
+        if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
+            url = getExportOVAUrlFromRemoteInstance(sourceInstance);
+        }
+        return url;
+    }
+
+    private String getExportOVAUrlFromRemoteInstance(RemoteInstanceTO vmwareInstance) {
+        String vcenter = vmwareInstance.getVcenterHost();
+        String username = vmwareInstance.getVcenterUsername();
+        String password = vmwareInstance.getVcenterPassword();
+        String datacenter = vmwareInstance.getDatacenterName();
+        String vm = vmwareInstance.getInstanceName();
+
+        String encodedUsername = encodeUsername(username);
+        String encodedPassword = encodeUsername(password);
+        return String.format("vi://%s:%s@%s/%s/vm/%s",
+                encodedUsername, encodedPassword, vcenter, datacenter, vm);
+    }
+
     protected List<KVMPhysicalDisk> getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) {
         List<LibvirtVMDef.DiskDef> disksDefs = xmlParser.getDisks();
         disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE &&
@@ -204,11 +256,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
         Script.runSimpleBashScript(String.format("rm -f %s/%s*.xml", temporaryStoragePool.getLocalPath(), temporaryConvertUuid));
     }
 
-    protected boolean isInstanceConversionSupportedOnHost() {
-        int exitValue = Script.runSimpleBashScriptForExitValue(checkIfConversionIsSupportedCommand);
-        return exitValue == 0;
-    }
-
     protected void sanitizeDisksPath(List<LibvirtVMDef.DiskDef> disks) {
         for (LibvirtVMDef.DiskDef disk : disks) {
             String[] diskPathParts = disk.getDiskPath().split("/");
@@ -234,6 +281,11 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
                 logger.error(err);
                 continue;
             }
+            if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) {
+                String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath);
+                logger.error(err);
+                continue;
+            }
             KVMPhysicalDisk sourceDisk = temporaryDisks.get(i);
             if (logger.isDebugEnabled()) {
                 String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" +
@@ -305,25 +357,45 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
         String sourceHostIp = null;
         String sourcePath = null;
         String storagePoolMountPoint = Script.runSimpleBashScript(String.format("mount | grep %s", storagePool.getLocalPath()));
+        logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint));
         if (StringUtils.isNotEmpty(storagePoolMountPoint)) {
             String[] res = storagePoolMountPoint.strip().split(" ");
             res = res[0].split(":");
+            if (res.length > 1) {
                 sourceHostIp = res[0].strip();
                 sourcePath = res[1].strip();
             }
+        }
         return new Pair<>(sourceHostIp, sourcePath);
     }
 
-    protected boolean performInstanceConversion(String convertInstanceUrl, String sourceInstanceName,
+    private boolean exportOVAFromVMOnVcenter(String vmExportUrl,
-                                              String temporaryPasswordFilePath,
+                                             String targetOvfDir,
+                                             int noOfThreads,
+                                             long timeout) {
+        Script script = new Script("ovftool", timeout, logger);
+        script.add("--noSSLVerify");
+        if (noOfThreads > 1) {
+            script.add(String.format("--parallelThreads=%s", noOfThreads));
+        }
+        script.add(vmExportUrl);
+        script.add(targetOvfDir);
+
+        String logPrefix = "export ovf";
+        OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix);
+        script.execute(outputLogger);
+        int exitValue = script.getExitValue();
+        return exitValue == 0;
+    }
+
+    protected boolean performInstanceConversion(String sourceOVFDirPath,
                                                 String temporaryConvertFolder,
                                                 String temporaryConvertUuid,
                                                 long timeout, boolean verboseModeEnabled) {
         Script script = new Script("virt-v2v", timeout, logger);
         script.add("--root", "first");
-        script.add("-ic", convertInstanceUrl);
+        script.add("-i", "ova");
-        script.add(sourceInstanceName);
+        script.add(sourceOVFDirPath);
-        script.add("--password-file", temporaryPasswordFilePath);
         script.add("-o", "local");
         script.add("-os", temporaryConvertFolder);
         script.add("-of", "qcow2");
@@ -332,44 +404,13 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
             script.add("-v");
         }
 
-        String logPrefix = String.format("virt-v2v source: %s %s progress", convertInstanceUrl, sourceInstanceName);
+        String logPrefix = String.format("virt-v2v ovf source: %s progress", sourceOVFDirPath);
         OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix);
         script.execute(outputLogger);
         int exitValue = script.getExitValue();
         return exitValue == 0;
     }
 
-    private String createTemporaryPasswordFileAndRetrievePath(RemoteInstanceTO sourceInstance) {
-        String password = null;
-        if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
-            password = sourceInstance.getVcenterPassword();
-        }
-        String passwordFile = String.format("/tmp/vmw-%s", UUID.randomUUID());
-        String msg = String.format("Creating a temporary password file for VMware instance %s conversion on: %s", sourceInstance.getInstanceName(), passwordFile);
-        logger.debug(msg);
-        Script.runSimpleBashScriptForExitValueAvoidLogging(String.format("echo \"%s\" > %s", password, passwordFile));
-        return passwordFile;
-    }
-
-    private String getConvertInstanceUrl(RemoteInstanceTO sourceInstance) {
-        String url = null;
-        if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
-            url = getConvertInstanceUrlFromVmware(sourceInstance);
-        }
-        return url;
-    }
-
-    private String getConvertInstanceUrlFromVmware(RemoteInstanceTO vmwareInstance) {
-        String vcenter = vmwareInstance.getVcenterHost();
-        String datacenter = vmwareInstance.getDatacenterName();
-        String username = vmwareInstance.getVcenterUsername();
-        String host = vmwareInstance.getHostName();
-        String cluster = vmwareInstance.getClusterName();
-
-        String encodedUsername = encodeUsername(username);
-        return String.format("vpx://%s@%s/%s/%s/%s?no_verify=1",
-                encodedUsername, vcenter, datacenter, cluster, host);
-    }
     protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException {
         String xmlPath = String.format("%s.xml", installPath);
         if (!new File(xmlPath).exists()) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.utils.cryptsetup.KeyFile;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
@@ -34,6 +35,7 @@ import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.commons.collections.CollectionUtils;
 import org.libvirt.Connect;
 import org.libvirt.LibvirtException;
 import org.libvirt.Secret;
@@ -283,9 +285,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
         }
     }
 
-    private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path) throws LibvirtException {
+    private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List<String> nfsMountOpts) throws LibvirtException {
         String targetPath = _mountPoint + File.separator + uuid;
-        LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath);
+        LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts);
         _storageLayer.mkdir(targetPath);
         StoragePool sp = null;
         try {
@@ -374,6 +376,42 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
 
     }
 
+    private List<String> getNFSMountOptsFromDetails(StoragePoolType type, Map<String, String> details) {
+        List<String> nfsMountOpts = null;
+        if (!type.equals(StoragePoolType.NetworkFilesystem) || details == null) {
+            return nfsMountOpts;
+        }
+        if (details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) {
+            nfsMountOpts = Arrays.asList(details.get(ApiConstants.NFS_MOUNT_OPTIONS).replaceAll("\\s", "").split(","));
+        }
+        return nfsMountOpts;
+    }
+
+    private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connect conn, List<String> nfsMountOpts) {
+        try {
+            LibvirtStoragePoolDef poolDef = getStoragePoolDef(conn, sp);
+            Set poolNfsMountOpts = poolDef.getNfsMountOpts();
+            boolean mountOptsDiffer = false;
+            if (poolNfsMountOpts.size() != nfsMountOpts.size()) {
+                mountOptsDiffer = true;
+            } else {
+                for (String nfsMountOpt : nfsMountOpts) {
+                    if (!poolNfsMountOpts.contains(nfsMountOpt)) {
+                        mountOptsDiffer = true;
+                        break;
+                    }
+                }
+            }
+            if (mountOptsDiffer) {
+                sp.destroy();
+                return true;
+            }
+        } catch (LibvirtException e) {
+            logger.error("Failure in destroying the pre-existing storage pool for changing the NFS mount options" + e);
+        }
+        return false;
+    }
+
     private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) {
 
         LibvirtStoragePoolDef spd;
@@ -671,12 +709,21 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
             } catch (LibvirtException e) {
                 logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e);
             }
+        }
+
+        List<String> nfsMountOpts = getNFSMountOptsFromDetails(type, details);
+        if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) &&
+            destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) {
+            sp = null;
+        }
+
+        if (sp == null) {
 
             logger.debug("Attempting to create storage pool " + name);
 
             if (type == StoragePoolType.NetworkFilesystem) {
                 try {
-                    sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path);
+                    sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts);
                 } catch (LibvirtException e) {
                     logger.error("Failed to create netfs mount: " + host + ":" + path , e);
                     logger.error(e.getStackTrace());
@@ -684,7 +731,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
                 }
             } else if (type == StoragePoolType.Gluster) {
                 try {
-                    sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path);
+                    sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null);
                 } catch (LibvirtException e) {
                     logger.error("Failed to create glusterfs mount: " + host + ":" + path , e);
                     logger.error(e.getStackTrace());

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java

@@ -19,6 +19,9 @@
 
 package com.cloud.hypervisor.kvm.resource;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import junit.framework.TestCase;
 import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.PoolType;
 import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.AuthenticationType;
@@ -47,6 +50,14 @@ public class LibvirtStoragePoolDefTest extends TestCase {
         assertEquals(port, pool.getSourcePort());
         assertEquals(dir, pool.getSourceDir());
         assertEquals(targetPath, pool.getTargetPath());
+
+        List<String> nfsMountOpts = new ArrayList<>();
+        nfsMountOpts.add("vers=4.1");
+        nfsMountOpts.add("nconnect=4");
+        pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts);
+        assertTrue(pool.getNfsMountOpts().contains("vers=4.1"));
+        assertTrue(pool.getNfsMountOpts().contains("nconnect=4"));
+        assertEquals(pool.getNfsMountOpts().size(), 2);
     }
 
     @Test
@@ -57,11 +68,37 @@ public class LibvirtStoragePoolDefTest extends TestCase {
         String host = "127.0.0.1";
         String dir  = "/export/primary";
         String targetPath = "/mnt/" + uuid;
+        List<String> nfsMountOpts = new ArrayList<>();
+        nfsMountOpts.add("vers=4.1");
+        nfsMountOpts.add("nconnect=4");
 
-        LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath);
+        LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts);
 
-        String expectedXml = "<pool type='" + type.toString() + "'>\n<name>" + name + "</name>\n<uuid>" + uuid + "</uuid>\n" +
+        String expectedXml = "<pool type='" + type.toString() + "' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0'>\n" +
+                             "<name>" +name + "</name>\n<uuid>" + uuid + "</uuid>\n" +
                              "<source>\n<host name='" + host + "'/>\n<dir path='" + dir + "'/>\n</source>\n<target>\n" +
+                             "<path>" + targetPath + "</path>\n</target>\n" +
+                             "<fs:mount_opts>\n<fs:option name='vers=4.1'/>\n<fs:option name='nconnect=4'/>\n</fs:mount_opts>\n</pool>\n";
+
+        assertEquals(expectedXml, pool.toString());
+    }
+
+    @Test
+    public void testGlusterFSStoragePool() {
+        PoolType type = PoolType.GLUSTERFS;
+        String name = "myGFSPool";
+        String uuid = "89a605bc-d470-4637-b3df-27388be452f5";
+        String host = "127.0.0.1";
+        String dir  = "/export/primary";
+        String targetPath = "/mnt/" + uuid;
+        List<String> nfsMountOpts = new ArrayList<>();
+
+        LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts);
+
+        String expectedXml = "<pool type='netfs'>\n" +
+                "<name>" +name + "</name>\n<uuid>" + uuid + "</uuid>\n" +
+                "<source>\n<host name='" + host + "'/>\n<dir path='" + dir + "'/>\n" +
+                "<format type='glusterfs'/>\n</source>\n<target>\n" +
                 "<path>" + targetPath + "</path>\n</target>\n</pool>\n";
 
         assertEquals(expectedXml, pool.toString());

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java

@@ -30,7 +30,7 @@ public class LibvirtStoragePoolXMLParserTest extends TestCase {
 
     @Test
     public void testParseNfsStoragePoolXML() {
-        String poolXML = "<pool type='netfs'>\n" +
+        String poolXML = "<pool type='netfs' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0'>\n" +
                 "  <name>feff06b5-84b2-3258-b5f9-1953217295de</name>\n" +
                 "  <uuid>feff06b5-84b2-3258-b5f9-1953217295de</uuid>\n" +
                 "  <capacity unit='bytes'>111111111</capacity>\n" +
@@ -49,12 +49,18 @@ public class LibvirtStoragePoolXMLParserTest extends TestCase {
                 "      <group>0</group>\n" +
                 "    </permissions>\n" +
                 "  </target>\n" +
+                "  <fs:mount_opts>\n" +
+                "    <fs:option name='nconnect=8'/>\n" +
+                "    <fs:option name='vers=4.1'/>\n" +
+                "    </fs:mount_opts>\n" +
                 "</pool>";
 
         LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser();
         LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML);
 
-        Assert.assertEquals("10.11.12.13", pool.getSourceHost());
+        assertEquals("10.11.12.13", pool.getSourceHost());
+        assertTrue(pool.getNfsMountOpts().contains("vers=4.1"));
+        assertTrue(pool.getNfsMountOpts().contains("nconnect=8"));
     }
 
     @Test

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java

@@ -0,0 +1,67 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CheckConvertInstanceCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LibvirtCheckConvertInstanceCommandWrapperTest {
+
+    @Spy
+    private LibvirtCheckConvertInstanceCommandWrapper checkConvertInstanceCommandWrapper = Mockito.spy(LibvirtCheckConvertInstanceCommandWrapper.class);
+
+    @Mock
+    private LibvirtComputingResource libvirtComputingResourceMock;
+
+    @Mock
+    CheckConvertInstanceCommand checkConvertInstanceCommandMock;
+
+    @Before
+    public void setUp() {
+    }
+
+    @Test
+    public void testCheckInstanceCommand_success() {
+        Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true);
+        Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock);
+        assertTrue(answer.getResult());
+    }
+
+    @Test
+    public void testCheckInstanceCommand_failure() {
+        Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(false);
+        Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock);
+        assertFalse(answer.getResult());
+        assertTrue(StringUtils.isNotBlank(answer.getDetails()));
+    }
+}

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java

@@ -71,12 +71,6 @@ public class LibvirtConvertInstanceCommandWrapperTest {
 
     private static final String secondaryPoolUrl = "nfs://192.168.1.1/secondary";
     private static final String vmName = "VmToImport";
-    private static final String hostName = "VmwareHost1";
-    private static final String vmwareVcenter = "192.168.1.2";
-    private static final String vmwareDatacenter = "Datacenter";
-    private static final String vmwareCluster = "Cluster";
-    private static final String vmwareUsername = "administrator@vsphere.local";
-    private static final String vmwarePassword = "password";
 
     @Before
     public void setUp() {
@@ -88,15 +82,6 @@ public class LibvirtConvertInstanceCommandWrapperTest {
         Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2));
     }
 
-    @Test
-    public void testIsInstanceConversionSupportedOnHost() {
-        try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
-            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtConvertInstanceCommandWrapper.checkIfConversionIsSupportedCommand)).thenReturn(0);
-            boolean supported = convertInstanceCommandWrapper.isInstanceConversionSupportedOnHost();
-            Assert.assertTrue(supported);
-        }
-    }
-
     @Test
     public void testAreSourceAndDestinationHypervisorsSupported() {
         boolean supported = convertInstanceCommandWrapper.areSourceAndDestinationHypervisorsSupported(Hypervisor.HypervisorType.VMware, Hypervisor.HypervisorType.KVM);
@@ -190,6 +175,7 @@ public class LibvirtConvertInstanceCommandWrapperTest {
         Mockito.when(destDisk.getPath()).thenReturn("xyz");
         Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid))
                 .thenReturn(destinationPool);
+        Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem);
         Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt()))
                 .thenReturn(destDisk);
 
@@ -243,21 +229,16 @@ public class LibvirtConvertInstanceCommandWrapperTest {
         RemoteInstanceTO remoteInstanceTO = Mockito.mock(RemoteInstanceTO.class);
         Mockito.when(remoteInstanceTO.getHypervisorType()).thenReturn(hypervisorType);
         Mockito.when(remoteInstanceTO.getInstanceName()).thenReturn(vmName);
-        Mockito.when(remoteInstanceTO.getHostName()).thenReturn(hostName);
-        Mockito.when(remoteInstanceTO.getVcenterHost()).thenReturn(vmwareVcenter);
-        Mockito.when(remoteInstanceTO.getDatacenterName()).thenReturn(vmwareDatacenter);
-        Mockito.when(remoteInstanceTO.getClusterName()).thenReturn(vmwareCluster);
-        Mockito.when(remoteInstanceTO.getVcenterUsername()).thenReturn(vmwareUsername);
-        Mockito.when(remoteInstanceTO.getVcenterPassword()).thenReturn(vmwarePassword);
         return remoteInstanceTO;
     }
 
-    private ConvertInstanceCommand getConvertInstanceCommand(RemoteInstanceTO remoteInstanceTO, Hypervisor.HypervisorType hypervisorType) {
+    private ConvertInstanceCommand getConvertInstanceCommand(RemoteInstanceTO remoteInstanceTO, Hypervisor.HypervisorType hypervisorType, boolean checkConversionSupport) {
         ConvertInstanceCommand cmd = Mockito.mock(ConvertInstanceCommand.class);
         Mockito.when(cmd.getSourceInstance()).thenReturn(remoteInstanceTO);
         Mockito.when(cmd.getDestinationHypervisorType()).thenReturn(hypervisorType);
         Mockito.when(cmd.getWait()).thenReturn(14400);
         Mockito.when(cmd.getConversionTemporaryLocation()).thenReturn(secondaryDataStore);
+        Mockito.when(cmd.getCheckConversionSupport()).thenReturn(checkConversionSupport);
         return cmd;
     }
 
@@ -265,8 +246,8 @@ public class LibvirtConvertInstanceCommandWrapperTest {
     public void testExecuteConvertUnsupportedOnTheHost() {
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             RemoteInstanceTO remoteInstanceTO = getRemoteInstanceTO(Hypervisor.HypervisorType.VMware);
-            ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM);
+            ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM, true);
-            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtConvertInstanceCommandWrapper.checkIfConversionIsSupportedCommand)).thenReturn(1);
+            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtComputingResource.INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD)).thenReturn(1);
             Answer answer = convertInstanceCommandWrapper.execute(cmd, libvirtComputingResourceMock);
             Assert.assertFalse(answer.getResult());
         }
@@ -276,8 +257,8 @@ public class LibvirtConvertInstanceCommandWrapperTest {
     public void testExecuteConvertUnsupportedHypervisors() {
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             RemoteInstanceTO remoteInstanceTO = getRemoteInstanceTO(Hypervisor.HypervisorType.XenServer);
-            ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM);
+            ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM, true);
-            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtConvertInstanceCommandWrapper.checkIfConversionIsSupportedCommand)).thenReturn(0);
+            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtComputingResource.INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD)).thenReturn(0);
             Answer answer = convertInstanceCommandWrapper.execute(cmd, libvirtComputingResourceMock);
             Assert.assertFalse(answer.getResult());
         }
@@ -286,7 +267,7 @@ public class LibvirtConvertInstanceCommandWrapperTest {
     @Test
     public void testExecuteConvertFailure() {
         RemoteInstanceTO remoteInstanceTO = getRemoteInstanceTO(Hypervisor.HypervisorType.VMware);
-        ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM);
+        ConvertInstanceCommand cmd = getConvertInstanceCommand(remoteInstanceTO, Hypervisor.HypervisorType.KVM, true);
         String localMountPoint = "/mnt/xyz";
         Mockito.when(temporaryPool.getLocalPath()).thenReturn(localMountPoint);
 
@@ -296,15 +277,15 @@ public class LibvirtConvertInstanceCommandWrapperTest {
                  Mockito.when(mock.getExitValue()).thenReturn(1);
              })
         ) {
-            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtConvertInstanceCommandWrapper.checkIfConversionIsSupportedCommand)).thenReturn(0);
+            Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true);
-            Mockito.when(Script.runSimpleBashScriptForExitValueAvoidLogging(Mockito.anyString())).thenReturn(0);
+            Mockito.when(Script.runSimpleBashScriptForExitValue(LibvirtComputingResource.INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD)).thenReturn(0);
+            Mockito.when(Script.runSimpleBashScriptForExitValue(Mockito.anyString())).thenReturn(0);
             Mockito.when(Script.runSimpleBashScript(Mockito.anyString())).thenReturn("");
 
             Answer answer = convertInstanceCommandWrapper.execute(cmd, libvirtComputingResourceMock);
             Assert.assertFalse(answer.getResult());
             Mockito.verify(convertInstanceCommandWrapper).performInstanceConversion(Mockito.anyString(),
-                    Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean());
+                    Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean());
         }
     }
-
 }

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.storage;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.libvirt.Connect;
+import org.libvirt.StoragePool;
+import org.libvirt.StoragePoolInfo;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.hypervisor.kvm.resource.LibvirtConnection;
+import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef;
+import com.cloud.storage.Storage;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LibvirtStorageAdaptorTest {
+
+    private MockedStatic<LibvirtConnection> libvirtConnectionMockedStatic;
+
+    private AutoCloseable closeable;
+
+    @Spy
+    static LibvirtStorageAdaptor libvirtStorageAdaptor = new LibvirtStorageAdaptor(null);
+
+    @Before
+    public void initMocks() {
+        closeable = MockitoAnnotations.openMocks(this);
+        libvirtConnectionMockedStatic = Mockito.mockStatic(LibvirtConnection.class);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        libvirtConnectionMockedStatic.close();
+        closeable.close();
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testCreateStoragePoolWithNFSMountOpts() throws Exception {
+        LibvirtStoragePoolDef.PoolType type = LibvirtStoragePoolDef.PoolType.NETFS;
+        String name = "Primary1";
+        String uuid = String.valueOf(UUID.randomUUID());
+        String host = "127.0.0.1";
+        String dir  = "/export/primary";
+        String targetPath = "/mnt/" + uuid;
+
+        String poolXml = "<pool type='" + type.toString() + "' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0'>\n" +
+                "<name>" +name + "</name>\n<uuid>" + uuid + "</uuid>\n" +
+                "<source>\n<host name='" + host + "'/>\n<dir path='" + dir + "'/>\n</source>\n<target>\n" +
+                "<path>" + targetPath + "</path>\n</target>\n" +
+                "<fs:mount_opts>\n<fs:option name='vers=4.1'/>\n<fs:option name='nconnect=4'/>\n</fs:mount_opts>\n</pool>\n";
+
+        Connect conn =  Mockito.mock(Connect.class);
+        StoragePool sp = Mockito.mock(StoragePool.class);
+        StoragePoolInfo spinfo = Mockito.mock(StoragePoolInfo.class);
+        Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn);
+        Mockito.when(conn.storagePoolLookupByUUIDString(uuid)).thenReturn(sp);
+        Mockito.when(sp.isActive()).thenReturn(1);
+        Mockito.when(sp.getXMLDesc(0)).thenReturn(poolXml);
+
+        Map<String, String> details = new HashMap<>();
+        details.put("nfsmountopts", "vers=4.1, nconnect=4");
+        KVMStoragePool pool = libvirtStorageAdaptor.createStoragePool(uuid, null, 0, dir, null, Storage.StoragePoolType.NetworkFilesystem, details);
+    }
+}

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java

@@ -18,6 +18,8 @@ package com.cloud.hypervisor.guru;
 
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 
+import java.io.File;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -27,10 +29,12 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.hypervisor.vmware.mo.DatastoreMO;
 import com.cloud.hypervisor.vmware.mo.HostMO;
 import com.cloud.hypervisor.vmware.util.VmwareClient;
 import com.cloud.hypervisor.vmware.util.VmwareHelper;
+import com.cloud.utils.script.Script;
 import com.cloud.vm.VmDetailConstants;
 import com.vmware.vim25.VirtualMachinePowerState;
 import org.apache.cloudstack.acl.ControlledEntity;
@@ -41,12 +45,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.storage.NfsMountManager;
 import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.DeleteCommand;
 import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
@@ -148,16 +154,22 @@ import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import com.google.gson.Gson;
+import com.vmware.vim25.DistributedVirtualPort;
+import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
+import com.vmware.vim25.DistributedVirtualSwitchPortCriteria;
 import com.vmware.vim25.ManagedObjectReference;
+import com.vmware.vim25.VMwareDVSPortSetting;
 import com.vmware.vim25.VirtualDevice;
 import com.vmware.vim25.VirtualDeviceBackingInfo;
 import com.vmware.vim25.VirtualDeviceConnectInfo;
 import com.vmware.vim25.VirtualDisk;
 import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo;
 import com.vmware.vim25.VirtualEthernetCard;
+import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
 import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import com.vmware.vim25.VirtualMachineRuntimeInfo;
+import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec;
 
 public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable {
     private static final Gson GSON = GsonHelper.getGson();
@@ -186,6 +198,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     @Inject DiskOfferingDao diskOfferingDao;
     @Inject PhysicalNetworkDao physicalNetworkDao;
     @Inject StoragePoolHostDao storagePoolHostDao;
+    @Inject NfsMountManager mountManager;
 
     protected VMwareGuru() {
         super();
@@ -531,11 +544,29 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     /**
      * Get pool ID from datastore UUID
      */
-    private Long getPoolIdFromDatastoreUuid(String datastoreUuid) {
+    private Long getPoolIdFromDatastoreUuid(long zoneId, String datastoreUuid) {
+        StoragePoolVO pool = null;
+        try {
             String poolUuid = UuidUtils.normalize(datastoreUuid);
-        StoragePoolVO pool = _storagePoolDao.findByUuid(poolUuid);
+            s_logger.info("Trying to find pool by UUID: " + poolUuid);
+            pool = _storagePoolDao.findByUuid(poolUuid);
+        } catch (CloudRuntimeException ex) {
+            s_logger.warn("Unable to get pool by datastore UUID: " + ex.getMessage());
+        }
         if (pool == null) {
-            throw new CloudRuntimeException("Couldn't find storage pool " + poolUuid);
+            s_logger.info("Trying to find pool by path: " + datastoreUuid);
+            pool = _storagePoolDao.findPoolByZoneAndPath(zoneId, datastoreUuid);
+        }
+        if (pool == null && datastoreUuid.startsWith("-iqn") && datastoreUuid.endsWith("-0")) {
+            String iScsiName = "/iqn" + datastoreUuid.substring(4, datastoreUuid.length() - 2) + "/0";
+            s_logger.info("Trying to find volume by iScsi name: " + iScsiName);
+            VolumeVO volumeVO = _volumeDao.findOneByIScsiName(iScsiName);
+            if (volumeVO != null) {
+                pool = _storagePoolDao.findById(volumeVO.getPoolId());
+            }
+        }
+        if (pool == null) {
+            throw new CloudRuntimeException("Couldn't find storage pool " + datastoreUuid);
         }
         return pool.getId();
     }
@@ -543,13 +574,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     /**
      * Get pool ID for disk
      */
-    private Long getPoolId(VirtualDisk disk) {
+    private Long getPoolId(long zoneId, VirtualDisk disk) {
         VirtualDeviceBackingInfo backing = disk.getBacking();
         checkBackingInfo(backing);
         VirtualDiskFlatVer2BackingInfo info = (VirtualDiskFlatVer2BackingInfo)backing;
         String[] fileNameParts = info.getFileName().split(" ");
         String datastoreUuid = StringUtils.substringBetween(fileNameParts[0], "[", "]");
-        return getPoolIdFromDatastoreUuid(datastoreUuid);
+        return getPoolIdFromDatastoreUuid(zoneId, datastoreUuid);
     }
 
     /**
@@ -586,12 +617,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     /**
      * Get template pool ID
      */
-    private Long getTemplatePoolId(VirtualMachineMO template) throws Exception {
+    private Long getTemplatePoolId(long zoneId, VirtualMachineMO template) throws Exception {
         VirtualMachineConfigSummary configSummary = template.getConfigSummary();
         String vmPathName = configSummary.getVmPathName();
         String[] pathParts = vmPathName.split(" ");
         String dataStoreUuid = pathParts[0].replace("[", "").replace("]", "");
-        return getPoolIdFromDatastoreUuid(dataStoreUuid);
+        return getPoolIdFromDatastoreUuid(zoneId, dataStoreUuid);
     }
 
     /**
@@ -641,14 +672,14 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     /**
      * Get template ID for VM being imported. If it is not found, it is created
      */
-    private Long getImportingVMTemplate(List<VirtualDisk> virtualDisks, DatacenterMO dcMo, String vmInternalName, Long guestOsId, long accountId, Map<VirtualDisk, VolumeVO> disksMapping, Backup backup) throws Exception {
+    private Long getImportingVMTemplate(List<VirtualDisk> virtualDisks, long zoneId, DatacenterMO dcMo, String vmInternalName, Long guestOsId, long accountId, Map<VirtualDisk, VolumeVO> disksMapping, Backup backup) throws Exception {
         for (VirtualDisk disk : virtualDisks) {
             if (isRootDisk(disk, disksMapping, backup)) {
                 VolumeVO volumeVO = disksMapping.get(disk);
                 if (volumeVO == null) {
                     String templatePath = getRootDiskTemplatePath(disk);
                     VirtualMachineMO template = getTemplate(dcMo, templatePath);
-                    Long poolId = getTemplatePoolId(template);
+                    Long poolId = getTemplatePoolId(zoneId, template);
                     Long templateSize = getTemplateSize(template, vmInternalName, disksMapping, backup);
                     long templateId = getTemplateId(templatePath, vmInternalName, guestOsId, accountId);
                     updateTemplateRef(templateId, poolId, templatePath, templateSize);
@@ -742,7 +773,11 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     protected VolumeVO updateVolume(VirtualDisk disk, Map<VirtualDisk, VolumeVO> disksMapping, VirtualMachineMO vmToImport, Long poolId, VirtualMachine vm) throws Exception {
         VolumeVO volume = disksMapping.get(disk);
         String volumeName = getVolumeName(disk, vmToImport);
+        if (volume.get_iScsiName() != null) {
+            volume.setPath(String.format("[%s] %s.vmdk", volumeName, volumeName));
+        } else {
             volume.setPath(volumeName);
+        }
         volume.setPoolId(poolId);
         VirtualMachineDiskInfo diskInfo = getDiskInfo(vmToImport, poolId, volumeName);
         volume.setChainInfo(GSON.toJson(diskInfo));
@@ -777,7 +812,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
 
         String operation = "";
         for (VirtualDisk disk : virtualDisks) {
-            Long poolId = getPoolId(disk);
+            Long poolId = getPoolId(zoneId, disk);
             Volume volume = null;
             if (disksMapping.containsKey(disk) && disksMapping.get(disk) != null) {
                 volume = updateVolume(disk, disksMapping, vmToImport, poolId, vmInstanceVO);
@@ -901,8 +936,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
         Map<String, NetworkVO> mapping = new HashMap<>();
         for (String networkName : vmNetworkNames) {
             NetworkVO networkVO = getGuestNetworkFromNetworkMorName(networkName, accountId, zoneId, domainId);
-            logger.debug(String.format("Mapping network name [%s] to networkVO [id: %s].", networkName, networkVO.getUuid()));
+            URI broadcastUri = networkVO.getBroadcastUri();
-            mapping.put(networkName, networkVO);
+            if (broadcastUri == null) {
+                continue;
+            }
+            String vlan = broadcastUri.getHost();
+            logger.debug(String.format("Mapping network vlan [%s] to networkVO [id: %s].", vlan, networkVO.getUuid()));
+            mapping.put(vlan, networkVO);
         }
         return mapping;
     }
@@ -912,7 +952,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
      */
     private NetworkMO getNetworkMO(VirtualEthernetCard nic, VmwareContext context) {
         VirtualDeviceConnectInfo connectable = nic.getConnectable();
-        VirtualEthernetCardNetworkBackingInfo info = (VirtualEthernetCardNetworkBackingInfo)nic.getBacking();
+        VirtualEthernetCardNetworkBackingInfo info = (VirtualEthernetCardNetworkBackingInfo) nic.getBacking();
         ManagedObjectReference networkMor = info.getNetwork();
         if (networkMor == null) {
             throw new CloudRuntimeException("Could not find network for NIC on: " + nic.getMacAddress());
@@ -920,22 +960,80 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
         return new NetworkMO(context, networkMor);
     }
 
-    private Pair<String, String> getNicMacAddressAndNetworkName(VirtualDevice nicDevice, VmwareContext context) throws Exception {
+    private Pair<String, String> getNicMacAddressAndVlan(VirtualDevice nicDevice, VmwareContext context) throws Exception {
         VirtualEthernetCard nic = (VirtualEthernetCard)nicDevice;
         String macAddress = nic.getMacAddress();
-        NetworkMO networkMO = getNetworkMO(nic, context);
+        VirtualDeviceBackingInfo backing = nic.getBacking();
-        String networkName = networkMO.getName();
+        if (backing instanceof VirtualEthernetCardNetworkBackingInfo) {
-        return new Pair<>(macAddress, networkName);
+            VirtualEthernetCardNetworkBackingInfo backingInfo = (VirtualEthernetCardNetworkBackingInfo) backing;
+            String deviceName = backingInfo.getDeviceName();
+            String vlan = getVlanFromDeviceName(deviceName);
+            return new Pair<>(macAddress, vlan);
+        } else if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) {
+            VirtualEthernetCardDistributedVirtualPortBackingInfo portInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing;
+            DistributedVirtualSwitchPortConnection port = portInfo.getPort();
+            String portKey = port.getPortKey();
+            String portGroupKey = port.getPortgroupKey();
+            String dvSwitchUuid = port.getSwitchUuid();
+            String vlan = getVlanFromDvsPort(context, dvSwitchUuid, portGroupKey, portKey);
+            return new Pair<>(macAddress, vlan);
+        }
+        return new Pair<>(macAddress, null);
+    }
+
+    private String getVlanFromDeviceName(String networkName) {
+        String prefix = "cloud.guest.";
+        if (!networkName.startsWith(prefix)) {
+            return null;
+        }
+        String nameWithoutPrefix = networkName.replace(prefix, "");
+        String[] parts = nameWithoutPrefix.split("\\.");
+        String vlan = parts[0];
+        return vlan;
+    }
+
+    private String getVlanFromDvsPort(VmwareContext context, String dvSwitchUuid, String portGroupKey, String portKey) {
+        try {
+            ManagedObjectReference dvSwitchManager = context.getVimClient().getServiceContent().getDvSwitchManager();
+            ManagedObjectReference dvSwitch = context.getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid);
+
+            // Get all ports
+            DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria();
+            criteria.setInside(true);
+            criteria.getPortgroupKey().add(portGroupKey);
+            List<DistributedVirtualPort> dvPorts = context.getVimClient().getService().fetchDVPorts(dvSwitch, criteria);
+
+            for (DistributedVirtualPort dvPort : dvPorts) {
+                if (!portKey.equals(dvPort.getKey())) {
+                    continue;
+                }
+                VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort.getConfig().getSetting();
+                VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan();
+                s_logger.debug("Found port " + dvPort.getKey() + " with vlan " + vlanId.getVlanId());
+                return String.valueOf(vlanId.getVlanId());
+            }
+        } catch (Exception ex) {
+            s_logger.error("Got exception while get vlan from DVS port: " + ex.getMessage());
+        }
+        return null;
     }
 
     private void syncVMNics(VirtualDevice[] nicDevices, DatacenterMO dcMo, Map<String, NetworkVO> networksMapping, VMInstanceVO vm) throws Exception {
         VmwareContext context = dcMo.getContext();
         List<NicVO> allNics = nicDao.listByVmId(vm.getId());
         for (VirtualDevice nicDevice : nicDevices) {
-            Pair<String, String> pair = getNicMacAddressAndNetworkName(nicDevice, context);
+            Pair<String, String> pair = getNicMacAddressAndVlan(nicDevice, context);
             String macAddress = pair.first();
-            String networkName = pair.second();
+            String vlanId = pair.second();
-            NetworkVO networkVO = networksMapping.get(networkName);
+            if (vlanId == null) {
+                s_logger.warn(String.format("vlanId for MAC address [%s] is null", macAddress));
+                continue;
+            }
+            NetworkVO networkVO = networksMapping.get(vlanId);
+            if (networkVO == null) {
+                s_logger.warn(String.format("Cannot find network for MAC address [%s] and vlanId [%s]", macAddress, vlanId));
+                continue;
+            }
             NicVO nicVO = nicDao.findByNetworkIdAndMacAddressIncludingRemoved(networkVO.getId(), macAddress);
             if (nicVO != null) {
                 logger.warn(String.format("Find NIC in DB with networkId [%s] and MAC Address [%s], so this NIC will be removed from list of unmapped NICs of VM [id: %s, name: %s].",
@@ -1066,7 +1164,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
 
         long guestOsId = getImportingVMGuestOs(configSummary);
         long serviceOfferingId = getImportingVMServiceOffering(configSummary, runtimeInfo);
-        long templateId = getImportingVMTemplate(virtualDisks, dcMo, vmInternalName, guestOsId, accountId, disksMapping, backup);
+        long templateId = getImportingVMTemplate(virtualDisks, zoneId, dcMo, vmInternalName, guestOsId, accountId, disksMapping, backup);
 
         VMInstanceVO vm = getVM(vmInternalName, templateId, guestOsId, serviceOfferingId, zoneId, accountId, userId, domainId);
         syncVMVolumes(vm, virtualDisks, disksMapping, vmToImport, backup);
@@ -1248,7 +1346,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
                                                      DatacenterMO dataCenterMO) throws Exception {
         HostMO sourceHost = vmMo.getRunningHost();
         String cloneName = UUID.randomUUID().toString();
-        DatastoreMO datastoreMO = vmMo.getAllDatastores().get(0); //pick the first datastore
+        List<DatastoreMO> vmDatastores = vmMo.getAllDatastores();
+        if (CollectionUtils.isEmpty(vmDatastores)) {
+            String err = String.format("Unable to fetch datastores, could not clone VM %s for migration from VMware", vmName);
+            s_logger.error(err);
+            throw new CloudRuntimeException(err);
+        }
+        DatastoreMO datastoreMO = vmDatastores.get(0); //pick the first datastore
         ManagedObjectReference morPool = vmMo.getRunningHost().getHyperHostOwnerResourcePool();
         boolean result = vmMo.createFullClone(cloneName, dataCenterMO.getVmFolder(), morPool, datastoreMO.getMor(), Storage.ProvisioningType.THIN);
         VirtualMachineMO clonedVM = dataCenterMO.findVm(cloneName);
@@ -1257,14 +1361,23 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
             logger.error(err);
             throw new CloudRuntimeException(err);
         }
+
         relocateClonedVMToSourceHost(clonedVM, sourceHost);
         return clonedVM;
     }
 
+    private String createOVFTemplateOfVM(VirtualMachineMO vmMO, DataStoreTO convertLocation, int threadsCountToExportOvf) throws Exception {
+        String dataStoreUrl = getDataStoreUrlForTemplate(convertLocation);
+        String vmOvfName = UUID.randomUUID().toString();
+        String vmOvfCreationPath = createDirOnStorage(vmOvfName, dataStoreUrl, null);
+        logger.debug(String.format("Creating OVF %s for the VM %s at %s", vmOvfName, vmMO.getName(), vmOvfCreationPath));
+        vmMO.exportVm(vmOvfCreationPath, vmOvfName, false, false, threadsCountToExportOvf);
+        logger.debug(String.format("Created OVF %s for the VM %s at %s", vmOvfName, vmMO.getName(), vmOvfCreationPath));
+        return vmOvfName;
+    }
+
     @Override
-    public UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName,
+    public Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params) {
-                                                                 Map<String, String> params) {
-        logger.debug(String.format("Cloning VM %s on external vCenter %s", vmName, hostIp));
         String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER_HOST);
         String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER_NAME);
         String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME);
@@ -1275,25 +1388,46 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
             DatacenterMO dataCenterMO = new DatacenterMO(context, datacenter);
             VirtualMachineMO vmMo = dataCenterMO.findVm(vmName);
             if (vmMo == null) {
-                String err = String.format("Cannot find VM with name %s on %s/%s", vmName, vcenter, datacenter);
+                String err = String.format("Cannot find VM with name %s on vCenter %s/%s", vmName, vcenter, datacenter);
                 logger.error(err);
                 throw new CloudRuntimeException(err);
             }
+
             VirtualMachinePowerState sourceVmPowerState = vmMo.getPowerState();
-            if (sourceVmPowerState == VirtualMachinePowerState.POWERED_ON && isWindowsVm(vmMo)) {
+
-                logger.debug(String.format("VM %s is a Windows VM and its Running, cannot be imported." +
+            if (sourceVmPowerState == VirtualMachinePowerState.POWERED_OFF) {
-                                "Please gracefully shut it down before attempting the import",
+                // Don't clone for powered off VMs, can export OVF from it
-                        vmName));
+                UnmanagedInstanceTO instanceTO = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
+                return new Pair<>(instanceTO, false);
             }
 
+            if (sourceVmPowerState == VirtualMachinePowerState.POWERED_ON) {
+                if (isWindowsVm(vmMo)) {
+                    String err = String.format("VM %s is a Windows VM and its Running, cannot be imported." +
+                            " Please gracefully shut it down before attempting the import", vmName);
+                    logger.error(err);
+                    throw new CloudRuntimeException(err);
+                }
+
+                if (isVMOnStandaloneHost(vmMo)) { // or datacenter.equalsIgnoreCase("ha-datacenter")? [Note: default datacenter name on standalone host: ha-datacenter]
+                    String err = String.format("VM %s might be on standalone host and is Running, cannot be imported." +
+                            " Please shut it down before attempting the import", vmName);
+                    logger.error(err);
+                    throw new CloudRuntimeException(err);
+                }
+            }
+
+            logger.debug(String.format("Cloning VM %s at VMware host %s on vCenter %s", vmName, hostIp, vcenter));
             VirtualMachineMO clonedVM = createCloneFromSourceVM(vmName, vmMo, dataCenterMO);
-            logger.debug(String.format("VM %s cloned successfully", vmName));
+            logger.debug(String.format("VM %s cloned successfully, to VM %s", vmName, clonedVM.getName()));
             UnmanagedInstanceTO clonedInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), clonedVM);
-            setNicsFromSourceVM(clonedInstance, vmMo);
+            setDisksFromSourceVM(clonedInstance, vmMo);
             clonedInstance.setCloneSourcePowerState(sourceVmPowerState == VirtualMachinePowerState.POWERED_ON ? UnmanagedInstanceTO.PowerState.PowerOn : UnmanagedInstanceTO.PowerState.PowerOff);
-            return clonedInstance;
+            return new Pair<>(clonedInstance, true);
+        } catch (CloudRuntimeException cre) {
+            throw cre;
         } catch (Exception e) {
-            String err = String.format("Error cloning VM: %s from external vCenter %s: %s", vmName, vcenter, e.getMessage());
+            String err = String.format("Error while finding or cloning VM: %s from vCenter %s: %s", vmName, vcenter, e.getMessage());
             logger.error(err, e);
             throw new CloudRuntimeException(err, e);
         }
@@ -1304,7 +1438,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
         return sourceInstance.getOperatingSystem().toLowerCase().contains("windows");
     }
 
-    private void setNicsFromSourceVM(UnmanagedInstanceTO clonedInstance, VirtualMachineMO vmMo) throws Exception {
+    private boolean isVMOnStandaloneHost(VirtualMachineMO vmMo) throws Exception {
+        UnmanagedInstanceTO sourceInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
+        return StringUtils.isEmpty(sourceInstance.getClusterName());
+    }
+
+    private void setDisksFromSourceVM(UnmanagedInstanceTO clonedInstance, VirtualMachineMO vmMo) throws Exception {
         UnmanagedInstanceTO sourceInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
         List<UnmanagedInstanceTO.Disk> sourceDisks = sourceInstance.getDisks();
         List<UnmanagedInstanceTO.Disk> clonedDisks = clonedInstance.getDisks();
@@ -1316,12 +1455,40 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     }
 
     @Override
-    public boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params) {
+    public String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf) {
-        logger.debug(String.format("Removing VM %s on external vCenter %s", vmName, hostIp));
         String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER_HOST);
         String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER_NAME);
         String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME);
         String password = params.get(VmDetailConstants.VMWARE_VCENTER_PASSWORD);
+        logger.debug(String.format("Creating template of the VM %s at VMware host %s on vCenter %s", vmName, hostIp, vcenter));
+
+        try {
+            VmwareContext context = connectToVcenter(vcenter, username, password);
+            DatacenterMO dataCenterMO = new DatacenterMO(context, datacenter);
+            VirtualMachineMO vmMo = dataCenterMO.findVm(vmName);
+            if (vmMo == null) {
+                String err = String.format("Cannot find VM with name %s on vCenter %s/%s, to create template file", vmName, vcenter, datacenter);
+                logger.error(err);
+                throw new CloudRuntimeException(err);
+            }
+            String ovaTemplate = createOVFTemplateOfVM(vmMo, templateLocation, threadsCountToExportOvf);
+            logger.debug(String.format("OVF %s created successfully on the datastore", ovaTemplate));
+            return ovaTemplate;
+        } catch (Exception e) {
+            String err = String.format("Error create template file of the VM: %s from vCenter %s: %s", vmName, vcenter, e.getMessage());
+            logger.error(err, e);
+            throw new CloudRuntimeException(err, e);
+        }
+    }
+
+    @Override
+    public boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params) {
+        String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER_HOST);
+        String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER_NAME);
+        String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME);
+        String password = params.get(VmDetailConstants.VMWARE_VCENTER_PASSWORD);
+        logger.debug(String.format("Removing cloned VM %s at VMware host %s on vCenter %s", vmName, hostIp, vcenter));
+
         try {
             VmwareContext context = connectToVcenter(vcenter, username, password);
             DatacenterMO dataCenterMO = new DatacenterMO(context, datacenter);
@@ -1332,11 +1499,97 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
                 logger.error(err);
                 return false;
             }
+
             return vmMo.destroy();
         } catch (Exception e) {
-            String err = String.format("Error destroying external VM %s: %s", vmName, e.getMessage());
+            String err = String.format("Error destroying cloned VM %s: %s", vmName, e.getMessage());
             logger.error(err, e);
             return false;
         }
     }
+
+    @Override
+    public boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir) {
+        logger.debug(String.format("Removing template %s", templateDir));
+
+        try {
+            String dataStoreUrl = getDataStoreUrlForTemplate(templateLocation);
+            return deleteDirOnStorage(templateDir, dataStoreUrl, null);
+        } catch (Exception e) {
+            String err = String.format("Error removing template file %s: %s", templateDir, e.getMessage());
+            logger.error(err, e);
+            return false;
+        }
+    }
+
+    private String getDataStoreUrlForTemplate(DataStoreTO templateLocation) {
+        String dataStoreUrl = null;
+        if (templateLocation instanceof NfsTO) {
+            NfsTO nfsStore = (NfsTO) templateLocation;
+            dataStoreUrl = nfsStore.getUrl();
+        } else if (templateLocation instanceof PrimaryDataStoreTO) {
+            PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) templateLocation;
+            if (primaryDataStoreTO.getPoolType().equals(Storage.StoragePoolType.NetworkFilesystem)) {
+                String psHost = primaryDataStoreTO.getHost();
+                String psPath = primaryDataStoreTO.getPath();
+                dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
+            }
+        }
+
+        if (dataStoreUrl == null) {
+            throw new CloudRuntimeException("Only NFS storage is supported for template creation");
+        }
+
+        return dataStoreUrl;
+    }
+
+    private String createDirOnStorage(String dirName, String nfsStorageUrl, String nfsVersion) throws Exception {
+        String mountPoint = mountManager.getMountPoint(nfsStorageUrl, nfsVersion);
+        logger.debug("Create dir storage location - url: " + nfsStorageUrl + ", mount point: " + mountPoint + ", dir: " + dirName);
+        String dirMountPath = mountPoint + File.separator + dirName;
+        createDir(dirMountPath);
+        return dirMountPath;
+    }
+
+    private void createDir(String dirName) throws Exception {
+        synchronized (dirName.intern()) {
+            Script command = new Script("mkdir", logger);
+            command.add("-p");
+            command.add(dirName);
+            String cmdResult = command.execute();
+            if (cmdResult != null) {
+                String msg = "Unable to create directory: " + dirName + ", error msg: " + cmdResult;
+                logger.error(msg);
+                throw new Exception(msg);
+            }
+        }
+    }
+
+    private boolean deleteDirOnStorage(String dirName, String nfsStorageUrl, String nfsVersion) throws Exception {
+        try {
+            String mountPoint = mountManager.getMountPoint(nfsStorageUrl, nfsVersion);
+            logger.debug("Delete dir storage location - url: " + nfsStorageUrl + ", mount point: " + mountPoint + ", dir: " + dirName);
+            String dirMountPath = mountPoint + File.separator + dirName;
+            deleteDir(dirMountPath);
+            return true;
+        } catch (Exception e) {
+            String err = String.format("Unable to delete dir %s: %s", dirName, e.getMessage());
+            logger.error(err, e);
+            return false;
+        }
+    }
+
+    private void deleteDir(String dirName) throws Exception {
+        synchronized (dirName.intern()) {
+            Script command = new Script("rm", logger);
+            command.add("-rf");
+            command.add(dirName);
+            String cmdResult = command.execute();
+            if (cmdResult != null) {
+                String msg = "Unable to delete directory: " + dirName + ", error msg: " + cmdResult;
+                logger.error(msg);
+                throw new Exception(msg);
+            }
+        }
+    }
 }

plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java

@@ -17,21 +17,37 @@
 package com.cloud.hypervisor.guru;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.cloudstack.storage.NfsMountManager;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.vm.UnmanagedInstanceTO;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
+import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
@@ -41,9 +57,19 @@ import org.springframework.test.context.support.AnnotationConfigContextLoader;
 
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.MigrateVmToPoolCommand;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.vmware.mo.DatacenterMO;
+import com.cloud.hypervisor.vmware.mo.DatastoreMO;
+import com.cloud.hypervisor.vmware.mo.HostMO;
+import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
+import com.cloud.hypervisor.vmware.util.VmwareClient;
+import com.cloud.hypervisor.vmware.util.VmwareContext;
+import com.cloud.hypervisor.vmware.util.VmwareHelper;
+import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ProvisioningType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.StoragePoolHostVO;
@@ -51,8 +77,15 @@ import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.utils.Pair;
+import com.cloud.utils.UuidUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.VmDetailConstants;
+import com.vmware.vim25.ManagedObjectReference;
+import com.vmware.vim25.ServiceContent;
+import com.vmware.vim25.VimPortType;
+import com.vmware.vim25.VirtualMachinePowerState;
 
 @RunWith(MockitoJUnitRunner.class)
 @ContextConfiguration(loader = AnnotationConfigContextLoader.class)
@@ -79,6 +112,21 @@ public class VMwareGuruTest {
 
     AutoCloseable closeable;
 
+    @Mock
+    NfsMountManager mountManager;
+
+    private static MockedStatic<VmwareHelper> mockedVmwareHelper;
+
+    @BeforeClass
+    public static void init() {
+        mockedVmwareHelper = Mockito.mockStatic(VmwareHelper.class);
+    }
+
+    @AfterClass
+    public static void close() {
+        mockedVmwareHelper.close();
+    }
+
     @Before
     public void testSetUp() throws Exception {
         closeable = MockitoAnnotations.openMocks(this);
@@ -155,4 +203,415 @@ public class VMwareGuruTest {
 
         assertEquals(expected, result);
     }
+
+    @Test(expected=CloudRuntimeException.class)
+    public void testCloneHypervisorVM_NoExternalVM() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(null).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
+        }
+    }
+
+    @Test(expected=CloudRuntimeException.class)
+    public void testCloneHypervisorVM_WindowsVMRunning() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        HostMO hostMo = Mockito.mock(HostMO.class);
+        Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
+        Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
+        UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
+        Mockito.doReturn("Windows 2019").when(instance).getOperatingSystem();
+        when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
+        }
+    }
+
+    @Test(expected=CloudRuntimeException.class)
+    public void testCloneHypervisorVM_GetDatastoresFailed() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        HostMO hostMo = Mockito.mock(HostMO.class);
+        Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
+        Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
+        UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
+        Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
+        Mockito.doReturn("test-cluster").when(instance).getClusterName();
+        when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
+        List<DatastoreMO> datastores = new ArrayList<>();
+        Mockito.doReturn(datastores).when(vmMo).getAllDatastores();
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
+        }
+    }
+
+    @Test(expected=CloudRuntimeException.class)
+    public void testCloneHypervisorVM_CloneVMFailed() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        HostMO hostMo = Mockito.mock(HostMO.class);
+        Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
+        Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
+        Mockito.doReturn(mor).when(hostMo).getHyperHostOwnerResourcePool();
+        UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
+        Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
+        Mockito.doReturn("test-cluster").when(instance).getClusterName();
+        when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
+        DatastoreMO datastoreMO = Mockito.mock(DatastoreMO.class);
+        Mockito.doReturn(mor).when(datastoreMO).getMor();
+        List<DatastoreMO> datastores = new ArrayList<>();
+        datastores.add(datastoreMO);
+        Mockito.doReturn(datastores).when(vmMo).getAllDatastores();
+        Mockito.lenient().doReturn(false).when(vmMo).createFullClone(anyString(), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(Storage.ProvisioningType.class));
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(anyString());
+            Mockito.doReturn(mor).when(mockDatacenterMO).getVmFolder();
+            Mockito.doReturn(mor).when(mockDatacenterMO).getMor();
+        })) {
+            vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
+        }
+    }
+
+    @Test
+    public void testCloneHypervisorVM() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        HostMO hostMo = Mockito.mock(HostMO.class);
+        Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
+        Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
+        Mockito.doReturn(mor).when(hostMo).getHyperHostOwnerResourcePool();
+        Mockito.doReturn(mor).when(hostMo).getMor();
+        DatastoreMO datastoreMO = Mockito.mock(DatastoreMO.class);
+        Mockito.doReturn(mor).when(datastoreMO).getMor();
+        List<DatastoreMO> datastores = new ArrayList<>();
+        datastores.add(datastoreMO);
+        Mockito.doReturn(datastores).when(vmMo).getAllDatastores();
+        Mockito.lenient().doReturn(true).when(vmMo).createFullClone(anyString(), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(Storage.ProvisioningType.class));
+        UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
+        Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
+        Mockito.doReturn("test-cluster").when(instance).getClusterName();
+        when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
+        UnmanagedInstanceTO.Disk disk = Mockito.mock(UnmanagedInstanceTO.Disk.class);
+        Mockito.doReturn("1").when(disk).getDiskId();
+        List<UnmanagedInstanceTO.Disk> disks = new ArrayList<>();
+        disks.add(disk);
+        Mockito.doReturn(disks).when(instance).getDisks();
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(anyString());
+            Mockito.doReturn(mor).when(mockDatacenterMO).getVmFolder();
+            Mockito.doReturn(mor).when(mockDatacenterMO).getMor();
+        })) {
+            Pair<UnmanagedInstanceTO, Boolean> clonedVm = vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
+            assertNotNull(clonedVm);
+        }
+    }
+
+    @Test(expected=CloudRuntimeException.class)
+    public void testCreateVMTemplateFileOutOfBand_NoClonedVM() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "cloned-test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        DataStoreTO dataStore = Mockito.mock(DataStoreTO.class);
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(null).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore, -1);
+        }
+    }
+
+    @Test
+    public void testCreateVMTemplateFileOutOfBand() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "cloned-test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        Mockito.doNothing().when(vmMo).exportVm(anyString(), anyString(), anyBoolean(), anyBoolean(), anyInt());
+        NfsTO dataStore = Mockito.mock(NfsTO.class);
+        Mockito.doReturn("nfs://10.1.1.4/testdir").when(dataStore).getUrl();
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            String vmTemplate = vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore, -1);
+            assertNotNull(vmTemplate);
+            assertTrue(UuidUtils.isUuid(vmTemplate));
+        }
+    }
+
+    @Test
+    public void testRemoveClonedHypervisorVM_NoClonedVM() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "cloned-test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(null).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            boolean result = vMwareGuru.removeClonedHypervisorVMOutOfBand(hostIp, vmName, params);
+            assertFalse(result);
+        }
+    }
+
+    @Test
+    public void testRemoveClonedHypervisorVM() throws Exception {
+        String vCenterHost = "10.1.1.2";
+        String datacenterName = "datacenter";
+        String hostIp = "10.1.1.3";
+        String vmName = "cloned-test-vm";
+        Map<String, String> params = new HashMap<>();
+        params.put(VmDetailConstants.VMWARE_VCENTER_HOST, vCenterHost);
+        params.put(VmDetailConstants.VMWARE_DATACENTER_NAME, datacenterName);
+        params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, "username");
+        params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, "password");
+
+        ManagedObjectReference mor = Mockito.mock(ManagedObjectReference.class);
+        ServiceContent serviceContent = Mockito.mock(ServiceContent.class);
+        VimPortType vimPort = Mockito.mock(VimPortType.class);
+        VmwareClient vimClient = spy(new VmwareClient(vCenterHost));
+        VmwareContext vmwareContext = spy(new VmwareContext(vimClient, vCenterHost));
+        Mockito.doReturn(vimClient).when(vmwareContext).getVimClient();
+        Mockito.doReturn(mor).when(vmwareContext).getRootFolder();
+        Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
+        DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
+        VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
+        Mockito.doReturn(true).when(vmMo).destroy();
+
+        try (MockedConstruction<VmwareClient> ignored1 = Mockito.mockConstruction(VmwareClient.class, withSettings().spiedInstance(vimClient), (mockVmwareClient, contextVmwareClient) -> {
+            Mockito.doReturn(vimPort).when(mockVmwareClient).getService();
+            Mockito.doReturn(serviceContent).when(mockVmwareClient).getServiceContent();
+            Mockito.doNothing().when(mockVmwareClient).connect(anyString(), anyString(), anyString());
+            Mockito.doReturn(mor).when(mockVmwareClient).getRootFolder();
+        }); MockedConstruction<VmwareContext> ignored2 = Mockito.mockConstruction(VmwareContext.class, withSettings().spiedInstance(vmwareContext), (mockVmwareContext, contextVmwareContext) -> {
+            Mockito.doReturn(vimClient).when(mockVmwareContext).getVimClient();
+            Mockito.doReturn(mor).when(mockVmwareContext).getRootFolder();
+        }); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
+            Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
+        })) {
+            boolean result = vMwareGuru.removeClonedHypervisorVMOutOfBand(hostIp, vmName, params);
+            assertTrue(result);
+        }
+    }
+
+    @Test
+    public void testRemoveVMTemplateFileOutOfBand() throws Exception {
+        NfsTO dataStore = Mockito.mock(NfsTO.class);
+        Mockito.doReturn("nfs://10.1.1.4/testdir").when(dataStore).getUrl();
+        String templateDir = "f887b7b3-3d1f-4a7d-93e5-3147f58866c6";
+        boolean result = vMwareGuru.removeVMTemplateOutOfBand(dataStore, templateDir);
+        assertTrue(result);
+    }
 }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -62,6 +62,7 @@ import org.apache.cloudstack.diagnostics.DiagnosticsService;
 import org.apache.cloudstack.hypervisor.xenserver.ExtraConfigurationUtility;
 import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer;
 import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand;
+import org.apache.cloudstack.storage.configdrive.ConfigDrive;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.cloudstack.utils.security.ParserUtils;
@@ -221,6 +222,8 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S
 
     private final static String VM_NAME_ISO_SUFFIX = "-ISO";
 
+    private final static String VM_NAME_CONFIGDRIVE_ISO_SUFFIX = "-CONFIGDRIVE-ISO";
+
     private final static String VM_FILE_ISO_SUFFIX = ".iso";
     public final static int DEFAULTDOMRSSHPORT = 3922;
 
@@ -1018,12 +1021,13 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S
     protected SR createIsoSRbyURI(final Connection conn, final URI uri, final String vmName, final boolean shared) {
         try {
             final Map<String, String> deviceConfig = new HashMap<String, String>();
+            final boolean isConfigDrive = uri.toString().endsWith(ConfigDrive.CONFIGDRIVEDIR);
             String path = uri.getPath();
             path = path.replace("//", "/");
             deviceConfig.put("location", uri.getHost() + ":" + path);
             final Host host = Host.getByUuid(conn, _host.getUuid());
             final SR sr = SR.create(conn, host, deviceConfig, new Long(0), uri.getHost() + path, "iso", "iso", "iso", shared, new HashMap<String, String>());
-            sr.setNameLabel(conn, vmName + "-ISO");
+            sr.setNameLabel(conn, vmName + (isConfigDrive ? VM_NAME_CONFIGDRIVE_ISO_SUFFIX: VM_NAME_ISO_SUFFIX));
             sr.setNameDescription(conn, deviceConfig.get("location"));
 
             sr.scan(conn);
@@ -2646,9 +2650,10 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S
         return scsiid;
     }
 
-    public SR getISOSRbyVmName(final Connection conn, final String vmName) {
+    public SR getISOSRbyVmName(final Connection conn, final String vmName, boolean isConfigDrive) {
         try {
-            final Set<SR> srs = SR.getByNameLabel(conn, vmName + "-ISO");
+            final Set<SR> srs = SR.getByNameLabel(conn, vmName +
+                    (isConfigDrive ? VM_NAME_CONFIGDRIVE_ISO_SUFFIX : VM_NAME_ISO_SUFFIX));
             if (srs.size() == 0) {
                 return null;
             } else if (srs.size() == 1) {
@@ -2695,9 +2700,20 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S
         } catch (final URISyntaxException e) {
             throw new CloudRuntimeException("isoURL is wrong: " + isoURL);
         }
-        isoSR = getISOSRbyVmName(conn, vmName);
+        isoSR = getISOSRbyVmName(conn, vmName, false);
         if (isoSR == null) {
             isoSR = createIsoSRbyURI(conn, uri, vmName, false);
+        } else {
+            try {
+                String description = isoSR.getNameDescription(conn);
+                if (description.endsWith(ConfigDrive.CONFIGDRIVEDIR)) {
+                    throw new CloudRuntimeException(String.format("VM %s already has %s ISO attached. Please " +
+                            "stop-start VM to allow attaching-detaching both ISOs", vmName, ConfigDrive.CONFIGDRIVEDIR));
+                }
+            } catch (XenAPIException | XmlRpcException e) {
+                throw new CloudRuntimeException(String.format("Unable to retrieve name description for the already " +
+                        "attached ISO on VM %s", vmName));
+            }
         }
 
         final String isoName = isoURL.substring(index + 1);
@@ -5665,7 +5681,7 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S
             logger.debug("Attaching config drive iso device for the VM " + vmName + " In host " + ipAddr);
             Set<VM> vms = VM.getByNameLabel(conn, vmName);
 
-            SR sr = getSRByNameLabel(conn, vmName + VM_NAME_ISO_SUFFIX);
+            SR sr = getSRByNameLabel(conn, vmName + VM_NAME_CONFIGDRIVE_ISO_SUFFIX);
             //Here you will find only two vdis with the <vmname>.iso.
             //one is from source host and second from dest host
             Set<VDI> vdis = VDI.getByNameLabel(conn, vmName + VM_FILE_ISO_SUFFIX);

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStopCommandWrapper.java

@@ -142,8 +142,10 @@ public final class CitrixStopCommandWrapper extends CommandWrapper<StopCommand,
                                 networks.add(vif.getNetwork(conn));
                             }
                             vm.destroy(conn);
-                            final SR sr = citrixResourceBase.getISOSRbyVmName(conn, command.getVmName());
+                            final SR sr = citrixResourceBase.getISOSRbyVmName(conn, command.getVmName(), false);
                             citrixResourceBase.removeSR(conn, sr);
+                            final SR configDriveSR = citrixResourceBase.getISOSRbyVmName(conn, command.getVmName(), true);
+                            citrixResourceBase.removeSR(conn, configDriveSR);
                             // Disable any VLAN networks that aren't used
                             // anymore
                             for (final Network network : networks) {

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml

@@ -36,6 +36,17 @@ write_files:
       exit 0
       fi
 
+      sysctl net.ipv4.conf.default.arp_announce=0
+      sysctl net.ipv4.conf.default.arp_ignore=0
+      sysctl net.ipv4.conf.all.arp_announce=0
+      sysctl net.ipv4.conf.all.arp_ignore=0
+      sysctl net.ipv4.conf.eth0.arp_announce=0
+      sysctl net.ipv4.conf.eth0.arp_ignore=0
+      sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf
+
       ISO_MOUNT_DIR=/mnt/k8sdisk
       BINARIES_DIR=${ISO_MOUNT_DIR}/
       K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml

@@ -56,6 +56,17 @@ write_files:
       exit 0
       fi
 
+      sysctl net.ipv4.conf.default.arp_announce=0
+      sysctl net.ipv4.conf.default.arp_ignore=0
+      sysctl net.ipv4.conf.all.arp_announce=0
+      sysctl net.ipv4.conf.all.arp_ignore=0
+      sysctl net.ipv4.conf.eth0.arp_announce=0
+      sysctl net.ipv4.conf.eth0.arp_ignore=0
+      sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf
+
       ISO_MOUNT_DIR=/mnt/k8sdisk
       BINARIES_DIR=${ISO_MOUNT_DIR}/
       K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml

@@ -36,6 +36,17 @@ write_files:
       exit 0
       fi
 
+      sysctl net.ipv4.conf.default.arp_announce=0
+      sysctl net.ipv4.conf.default.arp_ignore=0
+      sysctl net.ipv4.conf.all.arp_announce=0
+      sysctl net.ipv4.conf.all.arp_ignore=0
+      sysctl net.ipv4.conf.eth0.arp_announce=0
+      sysctl net.ipv4.conf.eth0.arp_ignore=0
+      sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf
+      sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf
+
       ISO_MOUNT_DIR=/mnt/k8sdisk
       BINARIES_DIR=${ISO_MOUNT_DIR}/
       K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/

plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java

@@ -412,6 +412,10 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
                 throw new CloudRuntimeException("Storage has already been added as local storage");
             } catch (Exception e) {
                 logger.warn("Unable to establish a connection between " + h + " and " + primarystore, e);
+                String reason = storageMgr.getStoragePoolMountFailureReason(e.getMessage());
+                if (reason != null) {
+                    throw new CloudRuntimeException(reason);
+                }
             }
         }
 
@@ -427,7 +431,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
 
     @Override
     public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) {
-        List<HostVO> hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(hypervisorType, scope.getScopeId());
+        List<HostVO> hosts = _resourceMgr.listAllUpHostsInOneZoneByHypervisor(hypervisorType, scope.getScopeId());
         logger.debug("In createPool. Attaching the pool to each of the hosts.");
         List<HostVO> poolHosts = new ArrayList<HostVO>();
         for (HostVO host : hosts) {
@@ -439,6 +443,10 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
                     throw new CloudRuntimeException("Storage has already been added as local storage to host: " + host.getName());
             } catch (Exception e) {
                 logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e);
+                String reason = storageMgr.getStoragePoolMountFailureReason(e.getMessage());
+                if (reason != null) {
+                    throw new CloudRuntimeException(reason);
+                }
             }
         }
         if (poolHosts.isEmpty()) {
@@ -459,8 +467,8 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
 
     @Override
     public boolean cancelMaintain(DataStore store) {
-        dataStoreHelper.cancelMaintain(store);
         storagePoolAutmation.cancelMaintain(store);
+        dataStoreHelper.cancelMaintain(store);
         return true;
     }
 

plugins/storage/volume/default/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImplTest.java

@@ -34,6 +34,7 @@ import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StorageManagerImpl;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.utils.exception.CloudRuntimeException;
 import junit.framework.TestCase;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -57,7 +58,6 @@ import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
@@ -171,4 +171,23 @@ public class CloudStackPrimaryDataStoreLifeCycleImplTest extends TestCase {
     public void testAttachCluster() throws Exception {
         Assert.assertTrue(_cloudStackPrimaryDataStoreLifeCycle.attachCluster(store, new ClusterScope(1L, 1L, 1L)));
     }
+
+    @Test
+    public void testAttachClusterException() throws Exception {
+        String exceptionString = "Mount failed due to incorrect mount options.";
+        String mountFailureReason = "Incorrect mount option specified.";
+
+        CloudRuntimeException exception = new CloudRuntimeException(exceptionString);
+        StorageManager storageManager = Mockito.mock(StorageManager.class);
+        Mockito.when(storageManager.connectHostToSharedPool(Mockito.anyLong(), Mockito.anyLong())).thenThrow(exception);
+        Mockito.when(storageManager.getStoragePoolMountFailureReason(exceptionString)).thenReturn(mountFailureReason);
+        ReflectionTestUtils.setField(_cloudStackPrimaryDataStoreLifeCycle, "storageMgr", storageManager);
+
+        try {
+            _cloudStackPrimaryDataStoreLifeCycle.attachCluster(store, new ClusterScope(1L, 1L, 1L));
+            Assert.fail();
+        } catch (Exception e) {
+           Assert.assertEquals(e.getMessage(), mountFailureReason);
+        }
+    }
 }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java

@@ -1192,7 +1192,7 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
 
     @Override
     public boolean canCopy(DataObject srcData, DataObject destData) {
-        DataStore srcStore = destData.getDataStore();
+        DataStore srcStore = srcData.getDataStore();
         DataStore destStore = destData.getDataStore();
         if ((srcStore.getRole() == DataStoreRole.Primary && (srcData.getType() == DataObjectType.TEMPLATE || srcData.getType() == DataObjectType.VOLUME))
                 && (destStore.getRole() == DataStoreRole.Primary && destData.getType() == DataObjectType.VOLUME)) {

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.datastore.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class StorPoolSnapshotDef implements Serializable {
+    private static final long serialVersionUID = 1L;
+    private String name;
+    private Integer deleteAfter;
+    private Map<String, String> tags;
+    private Boolean bind;
+    private Integer iops;
+    private String rename;
+    private transient String volumeName;
+
+    public StorPoolSnapshotDef(String volumeName, Integer deleteAfter, Map<String, String> tags) {
+        super();
+        this.volumeName = volumeName;
+        this.deleteAfter = deleteAfter;
+        this.tags = tags;
+    }
+
+    public StorPoolSnapshotDef(String name, Integer deleteAfter, Map<String, String> tags, Boolean bind, Integer iops,
+            String rename) {
+        super();
+        this.name = name;
+        this.deleteAfter = deleteAfter;
+        this.tags = tags;
+        this.bind = bind;
+        this.iops = iops;
+        this.rename = rename;
+    }
+
+    public String getName() {
+        return name;
+    }
+    public void setName(String name) {
+        this.name = name;
+    }
+    public Integer getDeleteAfter() {
+        return deleteAfter;
+    }
+    public void setDeleteAfter(Integer deleteAfter) {
+        this.deleteAfter = deleteAfter;
+    }
+    public Map<String, String> getTags() {
+        return tags;
+    }
+    public void setTags(Map<String, String> tags) {
+        this.tags = tags;
+    }
+    public Boolean getBind() {
+        return bind;
+    }
+    public void setBind(Boolean bind) {
+        this.bind = bind;
+    }
+    public Integer getIops() {
+        return iops;
+    }
+    public void setIops(Integer iops) {
+        this.iops = iops;
+    }
+    public String getRename() {
+        return rename;
+    }
+    public void setRename(String rename) {
+        this.rename = rename;
+    }
+
+    public String getVolumeName() {
+        return volumeName;
+    }
+
+    public void setVolumeName(String volumeName) {
+        this.volumeName = volumeName;
+    }
+
+}

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java

@@ -18,6 +18,7 @@
  */
 package org.apache.cloudstack.storage.datastore.driver;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -31,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
@@ -42,6 +44,7 @@ import org.apache.cloudstack.storage.command.CommandResult;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -86,6 +89,8 @@ import com.cloud.server.ResourceTag;
 import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ResizeVolumePayload;
+import com.cloud.storage.Snapshot;
+import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
@@ -94,6 +99,7 @@ import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeDetailVO;
 import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.SnapshotDetailsDao;
 import com.cloud.storage.dao.SnapshotDetailsVO;
 import com.cloud.storage.dao.StoragePoolHostDao;
@@ -133,9 +139,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
     @Inject
     private HostDao hostDao;
     @Inject
-    private ResourceTagDao _resourceTagDao;
+    private ResourceTagDao resourceTagDao;
     @Inject
-    private SnapshotDetailsDao _snapshotDetailsDao;
+    private SnapshotDetailsDao snapshotDetailsDao;
+    @Inject
+    private SnapshotDao snapshotDao;
     @Inject
     private SnapshotDataStoreDao snapshotDataStoreDao;
     @Inject
@@ -402,7 +410,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
             }
             try {
                 SpConnectionDesc conn = StorPoolUtil.getSpConnection(dataStore.getUuid(), dataStore.getId(), storagePoolDetailsDao, primaryStoreDao);
-
+                tryToSnapshotVolumeBeforeDelete(vinfo, dataStore, name, conn);
                 SpApiResponse resp = StorPoolUtil.volumeDelete(name, conn);
                 if (resp.getError() == null) {
                     updateStoragePool(dataStore.getId(), - vinfo.getSize());
@@ -432,6 +440,54 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         callback.complete(res);
     }
 
+    private void tryToSnapshotVolumeBeforeDelete(VolumeInfo vinfo, DataStore dataStore, String name, SpConnectionDesc conn) {
+        Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(dataStore.getId());
+        if (deleteAfter != null && deleteAfter > 0 && vinfo.getPassphraseId() == null) {
+            createTemporarySnapshot(vinfo, name, deleteAfter, conn);
+        } else {
+            StorPoolUtil.spLog("The volume [%s] is not marked to be snapshot. Check the global setting `storpool.delete.after.interval` or the volume is encrypted [%s]", name, deleteAfter, vinfo.getPassphraseId() != null);
+        }
+    }
+
+    private void createTemporarySnapshot(VolumeInfo vinfo, String name, Integer deleteAfter, SpConnectionDesc conn) {
+        Map<String, String> tags = new HashMap<>();
+        tags.put("cs", StorPoolUtil.DELAY_DELETE);
+        StorPoolSnapshotDef snapshot = new StorPoolSnapshotDef(name, deleteAfter, tags);
+        StorPoolUtil.spLog("Creating backup snapshot before delete the volume [%s]", vinfo.getName());
+        SpApiResponse snapshotResponse = StorPoolUtil.volumeSnapshot(snapshot, conn);
+        if (snapshotResponse.getError() == null) {
+            String snapshotName = StorPoolUtil.getSnapshotNameFromResponse(snapshotResponse, false, StorPoolUtil.GLOBAL_ID);
+            String snapshotPath = StorPoolUtil.devPath(snapshotName);
+            SnapshotVO snapshotVo = createSnapshotVo(vinfo, snapshotName);
+            createSnapshotOnPrimaryVo(vinfo, snapshotVo, snapshotPath);
+            SnapshotDetailsVO snapshotDetails = new SnapshotDetailsVO(snapshotVo.getId(), StorPoolUtil.SP_DELAY_DELETE, "~" + snapshotName, true);
+            snapshotDetailsDao.persist(snapshotDetails);
+        }
+    }
+
+    private void createSnapshotOnPrimaryVo(VolumeInfo vinfo, SnapshotVO snapshotVo, String snapshotPath) {
+        SnapshotDataStoreVO snapshotOnPrimaryVo = new SnapshotDataStoreVO();
+        snapshotOnPrimaryVo.setSnapshotId(snapshotVo.getId());
+        snapshotOnPrimaryVo.setDataStoreId(vinfo.getDataCenterId());
+        snapshotOnPrimaryVo.setRole(vinfo.getDataStore().getRole());
+        snapshotOnPrimaryVo.setVolumeId(vinfo.getId());
+        snapshotOnPrimaryVo.setSize(vinfo.getSize());
+        snapshotOnPrimaryVo.setPhysicalSize(vinfo.getSize());
+        snapshotOnPrimaryVo.setInstallPath(snapshotPath);
+        snapshotOnPrimaryVo.setState(ObjectInDataStoreStateMachine.State.Ready);
+        snapshotDataStoreDao.persist(snapshotOnPrimaryVo);
+    }
+
+    private SnapshotVO createSnapshotVo(VolumeInfo vinfo, String snapshotName) {
+        SnapshotVO snapshotVo = new SnapshotVO(vinfo.getDataCenterId(), vinfo.getAccountId(), vinfo.getDomainId(), vinfo.getId(),
+                vinfo.getDiskOfferingId(), snapshotName,
+                (short)Snapshot.Type.RECURRING.ordinal(), Snapshot.Type.RECURRING.name(),
+                vinfo.getSize(), vinfo.getMinIops(), vinfo.getMaxIops(), vinfo.getHypervisorType(), Snapshot.LocationType.PRIMARY);
+        snapshotVo.setState(com.cloud.storage.Snapshot.State.BackedUp);
+        snapshotVo = snapshotDao.persist(snapshotVo);
+        return snapshotVo;
+    }
+
     private void logDataObject(final String pref, DataObject data) {
         final DataStore dstore = data.getDataStore();
         String name = null;
@@ -474,7 +530,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         try {
             if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.VOLUME) {
                 SnapshotInfo sinfo = (SnapshotInfo)srcData;
-                final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao);
+                final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, snapshotDetailsDao);
 
                 VolumeInfo vinfo = (VolumeInfo)dstData;
                 final String volumeName = vinfo.getUuid();
@@ -492,9 +548,12 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                     StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), to.getUuid(), snapshotName, sinfo.getUuid());
                 } else if (resp.getError().getName().equals("objectDoesNotExist")) {
                     //check if snapshot is on secondary storage
-                    StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snopshot on secondary storage", snapshotName);
+                    StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snapshot on secondary storage", snapshotName);
                     SnapshotDataStoreVO snap = getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId());
-                    if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) {
+                    SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
+                    if (snapshotDetail != null) {
+                        err = String.format("Could not create volume from snapshot due to: %s", resp.getError());
+                    } else if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) {
                         resp = StorPoolUtil.volumeCreate(srcData.getUuid(), null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn);
                         if (resp.getError() == null) {
                             VolumeObjectTO dstTO = (VolumeObjectTO) dstData.getTO();
@@ -515,11 +574,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                                     err = String.format("Could not freeze Storpool volume %s. Error: %s", srcData.getUuid(), resp2.getError());
                                 } else {
                                     String name = StorPoolUtil.getNameFromResponse(resp, false);
-                                    SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
+                                    SnapshotDetailsVO snapshotDetails = snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
                                     if (snapshotDetails != null) {
                                         StorPoolHelper.updateSnapshotDetailsValue(snapshotDetails.getId(), StorPoolUtil.devPath(name), "snapshot");
                                     }else {
-                                        StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), _snapshotDetailsDao);
+                                        StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), snapshotDetailsDao);
                                     }
                                     resp = StorPoolUtil.volumeCreate(volumeName, StorPoolUtil.getNameFromResponse(resp, true), size, null, null, "volume", sinfo.getBaseVolume().getMaxIops(), conn);
                                     if (resp.getError() == null) {
@@ -549,8 +608,10 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                     err = String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, resp.getError());
                 }
             } else if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.SNAPSHOT) {
+                SnapshotInfo sinfo = (SnapshotInfo)srcData;
+                SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
                 // bypass secondary storage
-                if (StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
+                if (StorPoolConfigurationManager.BypassSecondaryStorage.value() || snapshotDetail != null) {
                     SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData.getTO();
                     answer = new CopyCmdAnswer(snapshot);
                 } else {
@@ -987,9 +1048,9 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                 SnapshotObjectTO snapTo = (SnapshotObjectTO)snapshot.getTO();
                 snapTo.setPath(StorPoolUtil.devPath(name.split("~")[1]));
                 answer = new CreateObjectAnswer(snapTo);
-                StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), _snapshotDetailsDao);
+                StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), snapshotDetailsDao);
                 //add primary storage of snapshot
-                StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), _snapshotDetailsDao);
+                StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), snapshotDetailsDao);
                 StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.takeSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", name, snapshot.getUuid(), volumeName, vinfo.getUuid());
             }
         } catch (Exception e) {
@@ -1004,7 +1065,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
     @Override
     public void revertSnapshot(final SnapshotInfo snapshot, final SnapshotInfo snapshotOnPrimaryStore, final AsyncCompletionCallback<CommandResult> callback) {
         final VolumeInfo vinfo = snapshot.getBaseVolume();
-        final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao);
+        final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, snapshotDetailsDao);
         final String volumeName = StorPoolStorageAdaptor.getVolumeNameFromPath(vinfo.getPath(), true);
         StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.revertSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", snapshotName, snapshot.getUuid(), volumeName, vinfo.getUuid());
         String err = null;
@@ -1059,7 +1120,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
     }
 
     private String getVcPolicyTag(Long vmId) {
-        ResourceTag resourceTag = vmId != null ? _resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null;
+        ResourceTag resourceTag = vmId != null ? resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null;
         return resourceTag != null ? resourceTag.getValue() : "";
     }
 

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java

@@ -28,18 +28,28 @@ import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
 import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.storage.dao.SnapshotDetailsDao;
+import com.cloud.storage.dao.SnapshotDetailsVO;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
@@ -52,6 +62,12 @@ public class StorPoolStatsCollector extends ManagerBase {
     private StoragePoolDetailsDao storagePoolDetailsDao;
     @Inject
     private ConfigurationDao configurationDao;
+    @Inject
+    private SnapshotDao snapshotDao;
+    @Inject
+    private SnapshotDataStoreDao snapshotDataStoreDao;
+    @Inject
+    private SnapshotDetailsDao snapshotDetailsDao;
 
     private ScheduledExecutorService executor;
 
@@ -67,7 +83,7 @@ public class StorPoolStatsCollector extends ManagerBase {
     public boolean start() {
         List<StoragePoolVO> spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
         if (CollectionUtils.isNotEmpty(spPools)) {
-            executor = Executors.newScheduledThreadPool(2,new NamedThreadFactory("StorPoolStatsCollector"));
+            executor = Executors.newScheduledThreadPool(3, new NamedThreadFactory("StorPoolStatsCollector"));
             long storageStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("storage.stats.interval"), 60000L);
             long volumeStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("volume.stats.interval"), 60000L);
 
@@ -77,6 +93,13 @@ public class StorPoolStatsCollector extends ManagerBase {
             if (StorPoolConfigurationManager.StorageStatsInterval.value() > 0 && storageStatsInterval > 0) {
                 executor.scheduleAtFixedRate(new StorPoolStorageStatsMonitorTask(), 120, StorPoolConfigurationManager.StorageStatsInterval.value(), TimeUnit.SECONDS);
             }
+            for (StoragePoolVO pool: spPools) {
+                Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(pool.getId());
+                if (deleteAfter != null && deleteAfter > 0) {
+                    executor.scheduleAtFixedRate(new StorPoolSnapshotsWithDelayDelete(), 120, StorPoolConfigurationManager.ListSnapshotsWithDeleteAfterInterval.value(), TimeUnit.SECONDS);
+                    break;
+                }
+            }
         }
 
         return true;
@@ -182,4 +205,90 @@ public class StorPoolStatsCollector extends ManagerBase {
             }
         }
     }
+
+    class StorPoolSnapshotsWithDelayDelete implements Runnable {
+
+        @Override
+        public void run() {
+            List<StoragePoolVO> spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
+            if (CollectionUtils.isNotEmpty(spPools)) {
+                Map<Long, StoragePoolVO> onePoolForZone = new HashMap<>();
+                for (StoragePoolVO storagePoolVO : spPools) {
+                    onePoolForZone.put(storagePoolVO.getDataCenterId(), storagePoolVO);
+                }
+                for (StoragePoolVO storagePool : onePoolForZone.values()) {
+                    List<SnapshotDetailsVO> snapshotsDetails = snapshotDetailsDao.findDetailsByZoneAndKey(storagePool.getDataCenterId(), StorPoolUtil.SP_DELAY_DELETE);
+                    if (CollectionUtils.isEmpty(snapshotsDetails)) {
+                        return;
+                    }
+                    Map<String, String> snapshotsWithDelayDelete = new HashMap<>();
+
+                    try {
+                        logger.debug(String.format("Collecting snapshots marked to be deleted for zone [%s]", storagePool.getDataCenterId()));
+                        JsonArray arr = StorPoolUtil.snapshotsListAllClusters(StorPoolUtil.getSpConnection(storagePool.getUuid(),
+                                storagePool.getId(), storagePoolDetailsDao, storagePoolDao));
+                         snapshotsWithDelayDelete.putAll(getSnapshotsMarkedForDeletion(arr));
+                         logger.debug(String.format("Found snapshot details [%s] and snapshots on StorPool with delay delete flag [%s]", snapshotsDetails, snapshotsWithDelayDelete));
+                         syncSnapshots(snapshotsDetails, snapshotsWithDelayDelete);
+                    } catch (Exception e) {
+                        logger.debug("Could not fetch the snapshots with delay delete flag " + e.getMessage());
+                    }
+                }
+            }
+        }
+
+        private void syncSnapshots(List<SnapshotDetailsVO> snapshotsDetails,
+                Map<String, String> snapshotsWithDelayDelete) {
+            for (SnapshotDetailsVO snapshotDetailsVO : snapshotsDetails) {
+                 if (!snapshotsWithDelayDelete.containsKey(snapshotDetailsVO.getValue())) {
+                     StorPoolUtil.spLog("The snapshot [%s] with delayDelete flag is no longer on StorPool. Removing it from CloudStack", snapshotDetailsVO.getValue());
+                     SnapshotDataStoreVO ss = snapshotDataStoreDao
+                             .findBySourceSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary);
+                     if (ss != null) {
+                         ss.setState(State.Destroyed);
+                         snapshotDataStoreDao.update(ss.getId(), ss);
+                     }
+                     SnapshotVO snap = snapshotDao.findById(snapshotDetailsVO.getResourceId());
+                     if (snap != null) {
+                         snap.setState(com.cloud.storage.Snapshot.State.Destroyed);
+                         snapshotDao.update(snap.getId(), snap);
+                     }
+                     snapshotDetailsDao.remove(snapshotDetailsVO.getId());
+                 }
+             }
+        }
+
+        private  Map<String, String> getSnapshotsMarkedForDeletion(JsonArray arr) {
+            for (JsonElement jsonElement : arr) {
+                JsonObject error = jsonElement.getAsJsonObject().getAsJsonObject("error");
+                if (error != null) {
+                    throw new CloudRuntimeException(String.format("Could not collect the snapshots marked for deletion from all storage nodes due to: [%s]", error));
+                }
+            }
+            Map<String, String> snapshotsWithDelayDelete = new HashMap<>();
+            for (JsonElement jsonElement : arr) {
+                JsonObject response = jsonElement.getAsJsonObject().getAsJsonObject("response");
+                if (response == null) {
+                    return snapshotsWithDelayDelete;
+                }
+                collectSnapshots(snapshotsWithDelayDelete, response);
+            }
+            logger.debug("Found snapshots on StorPool" + snapshotsWithDelayDelete);
+            return snapshotsWithDelayDelete;
+        }
+
+        private void collectSnapshots(Map<String, String> snapshotsWithDelayDelete, JsonObject response) {
+            JsonArray snapshots = response.getAsJsonObject().getAsJsonArray("data");
+            for (JsonElement snapshot : snapshots) {
+                String name = snapshot.getAsJsonObject().get("name").getAsString();
+                JsonObject tags = snapshot.getAsJsonObject().get("tags").getAsJsonObject();
+                if (!StringUtils.startsWith(name, "*") && StringUtils.containsNone(name, "@") && tags != null && !tags.entrySet().isEmpty()) {
+                    String tag = tags.getAsJsonPrimitive("cs").getAsString();
+                    if (tag != null && tag.equals(StorPoolUtil.DELAY_DELETE)) {
+                        snapshotsWithDelayDelete.put(name, tag);
+                    }
+                }
+            }
+        }
+    }
 }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java

@@ -28,6 +28,8 @@ import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
+
+import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
@@ -125,6 +127,15 @@ public class StorPoolUtil {
 
     public static final String SP_VOLUME_ON_CLUSTER = "SP_VOLUME_ON_CLUSTER";
 
+    private static final String DATA = "data";
+
+    private static final String CLUSTERS = "clusters";
+
+    public static final String SP_DELAY_DELETE = "SP_DELAY_DELETE";
+
+    public static final String DELAY_DELETE = "delayDelete";
+
+
     public static enum StorpoolRights {
         RO("ro"), RW("rw"), DETACH("detach");
 
@@ -417,27 +428,31 @@ public class StorPoolUtil {
     public static JsonArray snapshotsList(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/SnapshotsList", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonArray data = obj.getAsJsonArray("data");
+        return obj.getAsJsonArray(DATA);
-        return data;
+    }
+
+    public static JsonArray snapshotsListAllClusters(SpConnectionDesc conn) {
+        SpApiResponse resp = GET("MultiCluster/AllClusters/SnapshotsList", conn);
+        JsonObject obj = resp.fullJson.getAsJsonObject();
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     public static JsonArray volumesList(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/VolumesList", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonArray data = obj.getAsJsonArray("data");
+        return obj.getAsJsonArray(DATA);
-        return data;
     }
 
     public static JsonArray volumesSpace(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/AllClusters/VolumesSpace", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        return obj.getAsJsonObject("data").getAsJsonArray("clusters");
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     public static JsonArray templatesStats(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/AllClusters/VolumeTemplatesStatus", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        return obj.getAsJsonObject("data").getAsJsonArray("clusters");
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     private static boolean objectExists(SpApiError err) {
@@ -454,7 +469,7 @@ public class StorPoolUtil {
         if (resp.getError() != null && !objectExists(resp.getError())) {
             return null;
         }
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         return data.getAsJsonPrimitive("size").getAsLong();
     }
 
@@ -462,7 +477,7 @@ public class StorPoolUtil {
         SpApiResponse resp = GET("MultiCluster/Snapshot/" + name, conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
 
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
         return clusterId != null ? clusterId.getAsString() : null;
     }
@@ -471,7 +486,7 @@ public class StorPoolUtil {
         SpApiResponse resp = GET("MultiCluster/Volume/" + name, conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
 
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
         return clusterId != null ? clusterId.getAsString() : null;
     }
@@ -580,6 +595,10 @@ public class StorPoolUtil {
         return POST("MultiCluster/VolumeSnapshot/" + volumeName, json, conn);
     }
 
+    public static SpApiResponse volumeSnapshot(StorPoolSnapshotDef snapshot, SpConnectionDesc conn) {
+        return POST("MultiCluster/VolumeSnapshot/" + snapshot.getVolumeName(), snapshot, conn);
+    }
+
     public static SpApiResponse volumesGroupSnapshot(final List<VolumeObjectTO> volumeTOs, final String vmUuid,
             final String snapshotName, String csTag, SpConnectionDesc conn) {
         Map<String, Object> json = new LinkedHashMap<>();
@@ -639,7 +658,7 @@ public class StorPoolUtil {
 
     public static String getSnapshotNameFromResponse(SpApiResponse resp, boolean tildeNeeded, String globalIdOrRemote) {
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive(globalIdOrRemote);
+        JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive(globalIdOrRemote);
         String name = data != null ? data.getAsString() : null;
         name = name != null ? !tildeNeeded ? name : "~" + name : name;
         return name;
@@ -647,7 +666,7 @@ public class StorPoolUtil {
 
     public static String getNameFromResponse(SpApiResponse resp, boolean tildeNeeded) {
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive("name");
+        JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive("name");
         String name = data != null ? data.getAsString() : null;
         name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name;
         return name;

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java

@@ -149,18 +149,21 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
     public StrategyPriority canHandle(DataObject srcData, DataObject destData) {
         DataObjectType srcType = srcData.getType();
         DataObjectType dstType = destData.getType();
-        if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE
+        if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE) {
-                && StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
             SnapshotInfo sinfo = (SnapshotInfo) srcData;
             VolumeInfo volume = sinfo.getBaseVolume();
             StoragePoolVO storagePool = _storagePool.findById(volume.getPoolId());
             if (!storagePool.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) {
                 return StrategyPriority.CANT_HANDLE;
             }
+            SnapshotDetailsVO snapshotDetail = _snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
+            if (snapshotDetail != null) {
+                throw new CloudRuntimeException("Cannot create a template from the last snapshot of deleted volume. You can only restore the volume.");
+            }
             String snapshotName = StorPoolHelper.getSnapshotName(sinfo.getId(), sinfo.getUuid(), _snapshotStoreDao,
                     _snapshotDetailsDao);
             StorPoolUtil.spLog("StorPoolDataMotionStrategy.canHandle snapshot name=%s", snapshotName);
-            if (snapshotName != null) {
+            if (snapshotName != null && StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
                 return StrategyPriority.HIGHEST;
             }
         }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java

@@ -44,6 +44,16 @@ public class StorPoolConfigurationManager implements Configurable {
             "The interval in seconds to get StorPool template statistics",
             false);
 
+    public static final ConfigKey<Integer> DeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class,
+            "storpool.delete.after.interval", "0",
+            "The interval (in seconds) after the StorPool snapshot will be deleted",
+            false, ConfigKey.Scope.StoragePool);
+
+    public static final ConfigKey<Integer> ListSnapshotsWithDeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class,
+            "storpool.list.snapshots.delete.after.interval", "360",
+            "The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag",
+            false);
+
     @Override
     public String getConfigComponentName() {
         return StorPoolConfigurationManager.class.getSimpleName();
@@ -51,6 +61,6 @@ public class StorPoolConfigurationManager implements Configurable {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval };
+        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval };
     }
 }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java

@@ -117,6 +117,8 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
                 if (resp.getError() != null) {
                     final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError());
                     StorPoolUtil.spLog(err);
+                    markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp);
+                    throw new CloudRuntimeException(err);
                 } else {
                     res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId);
                     StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name);
@@ -130,6 +132,16 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
         return res;
     }
 
+    private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, SpApiResponse resp) {
+        if (resp.getError().getName().equals("objectDoesNotExist")) {
+            SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySourceSnapshot(snapshotId, DataStoreRole.Primary);
+            if (snapshotOnPrimary != null) {
+                snapshotOnPrimary.setState(State.Destroyed);
+                _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary);
+            }
+        }
+    }
+
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
         logger.debug(String.format("StorpoolSnapshotStrategy.canHandle: snapshot=%s, uuid=%s, op=%s", snapshot.getName(), snapshot.getUuid(), op));
@@ -167,7 +179,7 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
         boolean resultIsSet = false;
         try {
             while (snapshot != null &&
-                (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) {
+                (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error || snapshot.getState() == Snapshot.State.BackedUp)) {
                 SnapshotInfo child = snapshot.getChild();
 
                 if (child != null) {
@@ -331,9 +343,21 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
         } else {
             snapshotZoneDao.removeSnapshotFromZones(snapshotVO.getId());
         }
+        if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotId, null))) {
+            return true;
+        }
+        updateSnapshotToDestroyed(snapshotVO);
         return true;
     }
 
+    private List<SnapshotInfo> retrieveSnapshotEntries(long snapshotId, Long zoneId) {
+        return snapshotDataFactory.getSnapshots(snapshotId, zoneId);
+    }
+
+    private void updateSnapshotToDestroyed(SnapshotVO snapshotVo) {
+        snapshotVo.setState(Snapshot.State.Destroyed);
+        _snapshotDao.update(snapshotVo.getId(), snapshotVo);
+    }
 
     @Override
     public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) {

plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/command/LinkAccountToLdapCmd.java

@@ -33,6 +33,7 @@ import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.LinkAccountToLdapResponse;
 import org.apache.cloudstack.api.response.LinkDomainToLdapResponse;
+import org.apache.cloudstack.api.response.RoleResponse;
 import org.apache.cloudstack.ldap.LdapManager;
 import org.apache.cloudstack.ldap.LdapUser;
 import org.apache.cloudstack.ldap.NoLdapUserMatchingQueryException;
@@ -61,9 +62,12 @@ public class LinkAccountToLdapCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ADMIN, type = CommandType.STRING, required = false, description = "domain admin username in LDAP ")
     private String admin;
 
-    @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.INTEGER, required = true, description = "Type of the account to auto import. Specify 0 for user and 2 for "
+    @Parameter(name = ApiConstants.ACCOUNT_TYPE, type = CommandType.INTEGER, required = false, description = "Type of the account to auto import. Specify 0 for user and 2 for "
             + "domain admin")
-    private int accountType;
+    private Integer accountType;
+
+    @Parameter(name = ApiConstants.ROLE_ID, type = CommandType.UUID, entityType = RoleResponse.class, required = false, description = "Creates the account under the specified role.", since="4.19.1")
+    private Long roleId;
 
     @Inject
     private LdapManager _ldapManager;
@@ -132,7 +136,14 @@ public class LinkAccountToLdapCmd extends BaseCmd {
     }
 
     public Account.Type getAccountType() {
-        return Account.Type.getFromValue(accountType);
+        if (accountType == null) {
+            return RoleType.getAccountTypeByRole(roleService.findRole(roleId), null);
+        }
+        return RoleType.getAccountTypeByRole(roleService.findRole(roleId), Account.Type.getFromValue(accountType.intValue()));
+    }
+
+    public Long getRoleId() {
+        return RoleType.getRoleByAccountType(roleId, getAccountType());
     }
 
     @Override

plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java

@@ -449,11 +449,12 @@ public class LdapManagerImpl extends ComponentLifecycleBase implements LdapManag
         Validate.notEmpty(cmd.getLdapDomain(), "ldapDomain cannot be empty, please supply a GROUP or OU name");
         Validate.notNull(cmd.getType(), "type cannot be null. It should either be GROUP or OU");
         Validate.notEmpty(cmd.getLdapDomain(), "GROUP or OU name cannot be empty");
+        Validate.isTrue(cmd.getAccountType() != null || cmd.getRoleId() != null, "Either account type or role ID must be given");
 
         LinkType linkType = LdapManager.LinkType.valueOf(cmd.getType().toUpperCase());
         Account account = accountDao.findActiveAccount(cmd.getAccountName(),cmd.getDomainId());
         if (account == null) {
-            account = new AccountVO(cmd.getAccountName(), cmd.getDomainId(), null, cmd.getAccountType(), UUID.randomUUID().toString());
+            account = new AccountVO(cmd.getAccountName(), cmd.getDomainId(), null, cmd.getAccountType(), cmd.getRoleId(), UUID.randomUUID().toString());
             accountDao.persist((AccountVO)account);
         }
 

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -5162,7 +5162,7 @@ public class ApiResponseHelper implements ResponseGenerator {
         } else if (instance.getHostName() != null) {
             response.setHostName(instance.getHostName());
         }
-        response.setPowerState(instance.getPowerState().toString());
+        response.setPowerState((instance.getPowerState() != null)? instance.getPowerState().toString() : UnmanagedInstanceTO.PowerState.PowerUnknown.toString());
         response.setCpuCores(instance.getCpuCores());
         response.setCpuSpeed(instance.getCpuSpeed());
         response.setCpuCoresPerSocket(instance.getCpuCoresPerSocket());

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -2989,6 +2989,16 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         return new Pair<>(pools, pools.size());
     }
 
+    private void setPoolResponseNFSMountOptions(StoragePoolResponse poolResponse, Long poolId) {
+        if (Storage.StoragePoolType.NetworkFilesystem.toString().equals(poolResponse.getType()) &&
+                HypervisorType.KVM.toString().equals(poolResponse.getHypervisor())) {
+            StoragePoolDetailVO detail = _storagePoolDetailsDao.findDetail(poolId, ApiConstants.NFS_MOUNT_OPTIONS);
+            if (detail != null) {
+                poolResponse.setNfsMountOpts(detail.getValue());
+            }
+        }
+    }
+
     private ListResponse<StoragePoolResponse> createStoragesPoolResponse(Pair<List<StoragePoolJoinVO>, Integer> storagePools) {
         ListResponse<StoragePoolResponse> response = new ListResponse<>();
 
@@ -3010,6 +3020,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
                     poolResponse.setCaps(caps);
                 }
             }
+            setPoolResponseNFSMountOptions(poolResponse, poolUuidToIdMap.get(poolResponse.getId()));
         }
 
         response.setResponses(poolResponses, storagePools.second());
@@ -4818,7 +4829,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         boolean showRemovedISO = cmd.getShowRemoved();
         Account caller = CallContext.current().getCallingAccount();
 
-        boolean listAll = cmd.listAll();
+        boolean listAll = false;
         if (isoFilter != null && isoFilter == TemplateFilter.all) {
             if (caller.getType() == Account.Type.NORMAL) {
                 throw new InvalidParameterValueException("Filter " + TemplateFilter.all + " can be specified by admin only");

server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java

@@ -247,7 +247,7 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
                     host.getHypervisorType() == Hypervisor.HypervisorType.Custom)) {
                 //only kvm has the requirement to return host details
                 try {
-                    hostResponse.setDetails(hostDetails);
+                    hostResponse.setDetails(hostDetails, host.getHypervisorType());
                 } catch (Exception e) {
                     logger.debug("failed to get host details", e);
                 }

server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java

@@ -34,16 +34,21 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.query.QueryService;
 
+import com.cloud.api.ApiDBUtils;
 import com.cloud.api.ApiResponseHelper;
 import com.cloud.api.query.vo.SnapshotJoinVO;
+import com.cloud.storage.GuestOS;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.Volume.Type;
+import com.cloud.storage.VolumeVO;
 import com.cloud.user.Account;
 import com.cloud.user.AccountService;
 import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
+import com.cloud.vm.VMInstanceVO;
 
 public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<SnapshotJoinVO, SnapshotResponse> implements SnapshotJoinDao {
 
@@ -121,6 +126,16 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<Snapsh
             snapshotResponse.setVolumeName(snapshot.getVolumeName());
             snapshotResponse.setVolumeType(snapshot.getVolumeType().name());
             snapshotResponse.setVirtualSize(snapshot.getVolumeSize());
+            VolumeVO volume = ApiDBUtils.findVolumeById(snapshot.getVolumeId());
+            if (volume != null && volume.getVolumeType() == Type.ROOT && volume.getInstanceId() != null) {
+                VMInstanceVO vm = ApiDBUtils.findVMInstanceById(volume.getInstanceId());
+                if (vm != null) {
+                    GuestOS guestOS = ApiDBUtils.findGuestOSById(vm.getGuestOSId());
+                    if (guestOS != null) {
+                        snapshotResponse.setOsTypeId(guestOS.getUuid());
+                    }
+                }
+            }
         }
         snapshotResponse.setZoneId(snapshot.getDataCenterUuid());
         snapshotResponse.setZoneName(snapshot.getDataCenterName());

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -474,9 +474,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
     private long _defaultPageSize = Long.parseLong(Config.DefaultPageSize.getDefaultValue());
     private static final String DOMAIN_NAME_PATTERN = "^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\\.)+[A-Za-z]{1,63}$";
-    protected Set<String> configValuesForValidation;
+    private Set<String> configValuesForValidation = new HashSet<String>();
-    private Set<String> weightBasedParametersForValidation;
+    private Set<String> weightBasedParametersForValidation = new HashSet<String>();
-    private Set<String> overprovisioningFactorsForValidation;
+    private Set<String> overprovisioningFactorsForValidation = new HashSet<String>();
 
     public static final ConfigKey<Boolean> SystemVMUseLocalStorage = new ConfigKey<Boolean>(Boolean.class, "system.vm.use.local.storage", "Advanced", "false",
             "Indicates whether to use local storage pools or shared storage pools for system VMs.", false, ConfigKey.Scope.Zone, null);
@@ -513,6 +513,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public static final ConfigKey<Boolean> ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS = new ConfigKey<>(Boolean.class, "allow.domain.admins.to.create.tagged.offerings", "Advanced",
             "false", "Allow domain admins to create offerings with tags.", true, ConfigKey.Scope.Account, null);
 
+    public static final ConfigKey<Long> DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0",
+            "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," +
+                    " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " +
+                    "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true);
+
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";
     private static final String BYTES_READ_RATE = "Bytes Read";
@@ -537,8 +542,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         return true;
     }
 
-    private void populateConfigValuesForValidationSet() {
+    protected void populateConfigValuesForValidationSet() {
-        configValuesForValidation = new HashSet<String>();
         configValuesForValidation.add("event.purge.interval");
         configValuesForValidation.add("account.cleanup.interval");
         configValuesForValidation.add("alert.wait");
@@ -566,10 +570,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         configValuesForValidation.add(StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.key());
         configValuesForValidation.add(UserDataManager.VM_USERDATA_MAX_LENGTH_STRING);
         configValuesForValidation.add(UnmanagedVMsManager.RemoteKvmInstanceDisksCopyTimeout.key());
+        configValuesForValidation.add(UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.key());
     }
 
     private void weightBasedParametersForValidation() {
-        weightBasedParametersForValidation = new HashSet<String>();
         weightBasedParametersForValidation.add(AlertManager.CPUCapacityThreshold.key());
         weightBasedParametersForValidation.add(AlertManager.StorageAllocatedCapacityThreshold.key());
         weightBasedParametersForValidation.add(AlertManager.StorageCapacityThreshold.key());
@@ -589,11 +593,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         weightBasedParametersForValidation.add(CapacityManager.SecondaryStorageCapacityThreshold.key());
         weightBasedParametersForValidation.add(ClusterDrsService.ClusterDrsImbalanceThreshold.key());
         weightBasedParametersForValidation.add(ClusterDrsService.ClusterDrsImbalanceSkipThreshold.key());
-
     }
 
     private void overProvisioningFactorsForValidation() {
-        overprovisioningFactorsForValidation = new HashSet<String>();
         overprovisioningFactorsForValidation.add(CapacityManager.MemOverprovisioningFactor.key());
         overprovisioningFactorsForValidation.add(CapacityManager.CpuOverprovisioningFactor.key());
         overprovisioningFactorsForValidation.add(CapacityManager.StorageOverprovisioningFactor.key());
@@ -1180,8 +1182,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         return new Pair<Configuration, String>(_configDao.findByName(name), newValue);
     }
 
-    private String validateConfigurationValue(final String name, String value, final String scope) {
+    protected String validateConfigurationValue(final String name, String value, final String scope) {
-
         final ConfigurationVO cfg = _configDao.findByName(name);
         if (cfg == null) {
             logger.error("Missing configuration variable " + name + " in configuration table");
@@ -1263,23 +1264,24 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             return null;
         }
 
-        if (type.equals(Integer.class) && NetworkModel.MACIdentifier.key().equalsIgnoreCase(name)) {
+        if (type.equals(Integer.class)) {
             try {
                 final int val = Integer.parseInt(value);
+                if (NetworkModel.MACIdentifier.key().equalsIgnoreCase(name)) {
                     //The value need to be between 0 to 255 because the mac generation needs a value of 8 bit
                     //0 value is considered as disable.
                     if(val < 0 || val > 255){
-                    throw new InvalidParameterValueException(name+" value should be between 0 and 255. 0 value will disable this feature");
+                        throw new InvalidParameterValueException(name + " value should be between 0 and 255. 0 value will disable this feature");
-                }
-            } catch (final NumberFormatException e) {
-                logger.error("There was an error trying to parse the integer value for:" + name);
-                throw new InvalidParameterValueException("There was an error trying to parse the integer value for:" + name);
                     }
                 }
 
-        if (type.equals(Integer.class) && configValuesForValidation.contains(name)) {
+                if (UnmanagedVMsManager.ThreadsOnMSToImportVMwareVMFiles.key().equalsIgnoreCase(name) || UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles.key().equalsIgnoreCase(name)) {
-            try {
+                    if (val > 10) {
-                final int val = Integer.parseInt(value);
+                        throw new InvalidParameterValueException("Please enter a value between 0 and 10 for the configuration parameter: " + name + ", -1 will disable it");
+                    }
+                }
+
+                if (configValuesForValidation.contains(name)) {
                     if (val <= 0) {
                         throw new InvalidParameterValueException("Please enter a positive value for the configuration parameter:" + name);
                     }
@@ -1299,9 +1301,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                             throw new InvalidParameterValueException("Please enter a value less than 1048576 for the configuration parameter:" + name);
                         }
                     }
+                }
             } catch (final NumberFormatException e) {
-                logger.error("There was an error trying to parse the integer value for:" + name);
+                logger.error("There was an error trying to parse the integer value for configuration parameter: " + name);
-                throw new InvalidParameterValueException("There was an error trying to parse the integer value for:" + name);
+                throw new InvalidParameterValueException("There was an error trying to parse the integer value for configuration parameter: " + name);
             }
         }
 
@@ -1312,8 +1315,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                     throw new InvalidParameterValueException("Please enter a value between 0 and 1 for the configuration parameter: " + name);
                 }
             } catch (final NumberFormatException e) {
-                logger.error("There was an error trying to parse the float value for:" + name);
+                logger.error("There was an error trying to parse the float value for configuration parameter: " + name);
-                throw new InvalidParameterValueException("There was an error trying to parse the float value for:" + name);
+                throw new InvalidParameterValueException("There was an error trying to parse the float value for configuration parameter: " + name);
             }
         }
 
@@ -4799,6 +4802,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             newIp6Gateway = MoreObjects.firstNonNull(newIp6Gateway, network.getIp6Gateway());
             newIp6Cidr = MoreObjects.firstNonNull(newIp6Cidr, network.getIp6Cidr());
             _networkModel.checkIp6Parameters(newIp6StartIp, newIp6EndIp, newIp6Gateway, newIp6Cidr);
+            if (!GuestType.Shared.equals(network.getGuestType())) {
+                _networkModel.checkIp6CidrSizeEqualTo64(newIp6Cidr);
+            }
             return true;
         }
         return false;
@@ -5277,6 +5283,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         endIpv6 = ObjectUtils.allNull(endIpv6, currentEndIPv6) ? null : MoreObjects.firstNonNull(endIpv6, currentEndIPv6);
 
         _networkModel.checkIp6Parameters(startIpv6, endIpv6, ip6Gateway, ip6Cidr);
+        final Network network = _networkModel.getNetwork(vlanRange.getNetworkId());
+        if (!GuestType.Shared.equals(network.getGuestType())) {
+            _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr);
+        }
 
         if (!ObjectUtils.allNull(startIpv6, endIpv6) && ObjectUtils.anyNull(startIpv6, endIpv6)) {
             throw new InvalidParameterValueException(String.format("Invalid IPv6 range %s-%s", startIpv6, endIpv6));
@@ -7862,9 +7872,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {SystemVMUseLocalStorage, IOPS_MAX_READ_LENGTH, IOPS_MAX_WRITE_LENGTH,
-                BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE, VM_SERVICE_OFFERING_MAX_CPU_CORES,
+                BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE,
-                VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS, ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN,
+                VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS,
-                ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN, ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, AllowNonRFC1918CompliantIPs
+                ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN,
+                ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE
         };
     }
 

server/src/main/java/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java

@@ -22,7 +22,6 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
 import org.apache.cloudstack.consoleproxy.ConsoleAccessManager;
-
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.framework.security.keys.KeysManager;
 import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
@@ -31,6 +30,7 @@ import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.GetVncPortAnswer;
 import com.cloud.agent.api.GetVncPortCommand;
 import com.cloud.agent.api.StartupProxyCommand;
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.info.ConsoleProxyInfo;
@@ -40,7 +40,9 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.vm.ConsoleProxyVO;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.dao.ConsoleProxyDao;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -180,6 +182,11 @@ public class AgentBasedConsoleProxyManager extends ManagerBase implements Consol
         return null;
     }
 
+    @Override
+    public void startProxyForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+            DeploymentPlanner planner) {
+    }
+
     @Override
     public boolean destroyProxy(long proxyVmId) {
         return false;

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java

@@ -16,11 +16,20 @@
 // under the License.
 package com.cloud.consoleproxy;
 
-import com.cloud.utils.component.Manager;
+import java.util.Map;
-import com.cloud.vm.ConsoleProxyVO;
 
 import org.apache.cloudstack.framework.config.ConfigKey;
 
+import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.utils.component.Manager;
+import com.cloud.vm.ConsoleProxyVO;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
+
 public interface ConsoleProxyManager extends Manager, ConsoleProxyService {
 
     int DEFAULT_PROXY_CAPACITY = 50;
@@ -53,6 +62,10 @@ public interface ConsoleProxyManager extends Manager, ConsoleProxyService {
 
     ConsoleProxyVO startProxy(long proxyVmId, boolean ignoreRestartSetting);
 
+    void startProxyForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params, DeploymentPlanner planner)
+            throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
+            OperationTimedoutException;
+
     boolean stopProxy(long proxyVmId);
 
     boolean rebootProxy(long proxyVmId);

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -74,6 +74,9 @@ import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.HostPodDao;
 import com.cloud.deploy.DataCenterDeployment;
 import com.cloud.deploy.DeployDestination;
+import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.event.ActionEvent;
+import com.cloud.event.EventTypes;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
@@ -491,6 +494,14 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
         return null;
     }
 
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_PROXY_START, eventDescription = "restarting console proxy VM for HA", async = true)
+    public void startProxyForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+            DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException {
+        virtualMachineManager.advanceStart(vm.getUuid(), params, planner);
+    }
+
     public ConsoleProxyVO assignProxyFromRunningPool(long dataCenterId) {
 
         if (logger.isDebugEnabled()) {

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -28,6 +28,8 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
@@ -64,13 +66,14 @@ import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.VpcVirtualNetworkApplianceService;
 import com.cloud.resource.ResourceManager;
 import com.cloud.server.ManagementServer;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.VolumeVO;
-import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.dao.GuestOSCategoryDao;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.storage.dao.VolumeDao;
@@ -79,6 +82,7 @@ import com.cloud.user.AccountManager;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.UserVmManager;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
@@ -142,6 +146,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
     VolumeDao volumeDao;
     @Inject
     DataStoreProviderManager dataStoreProviderMgr;
+    @Inject
+    VpcVirtualNetworkApplianceService routerService;
+    @Inject
+    UserVmManager userVmManager;
 
     long _serverId;
 
@@ -435,6 +443,36 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
 
     }
 
+    private void startVm(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+           DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException {
+        CallContext ctx = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine);
+        ctx.setEventResourceId(vm.getId());
+        try {
+            switch (vm.getType()) {
+                case DomainRouter:
+                    ctx.setEventResourceType(ApiCommandResourceType.DomainRouter);
+                    routerService.startRouterForHA(vm, params, planner);
+                    break;
+                case ConsoleProxy:
+                    ctx.setEventResourceType(ApiCommandResourceType.ConsoleProxy);
+                    consoleProxyManager.startProxyForHA(vm, params, planner);
+                    break;
+                case SecondaryStorageVm:
+                    ctx.setEventResourceType(ApiCommandResourceType.SystemVm);
+                    secondaryStorageVmManager.startSecStorageVmForHA(vm, params, planner);
+                    break;
+                case User:
+                    userVmManager.startVirtualMachineForHA(vm, params, planner);
+                    break;
+                default:
+                    _itMgr.advanceStart(vm.getUuid(), params, planner);
+            }
+        } finally {
+            CallContext.unregister();
+        }
+    }
+
     protected Long restart(final HaWorkVO work) {
         logger.debug("RESTART with HAWORK");
         List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
@@ -626,10 +664,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
                     }
                 }
                 // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency.
-                _itMgr.advanceStart(vm.getUuid(), params, null);
+                startVm(vm, params, null);
-            }catch (InsufficientCapacityException e){
+            } catch (InsufficientCapacityException e){
                 logger.warn("Failed to deploy vm " + vmId + " with original planner, sending HAPlanner");
-                _itMgr.advanceStart(vm.getUuid(), params, _haPlanners.get(0));
+                startVm(vm, params, _haPlanners.get(0));
             }
 
             VMInstanceVO started = _instanceDao.findById(vm.getId());
@@ -651,7 +689,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
         } catch (final ResourceUnavailableException e) {
             logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
             _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " +
-                hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
+                hostDesc, "The resource is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         } catch (ConcurrentOperationException e) {
             logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
             _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " +
@@ -659,7 +697,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
         } catch (OperationTimedoutException e) {
             logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
             _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " +
-                hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
+                    hostDesc, "The operation timed out while trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         }
         vm = _itMgr.findById(vm.getId());
         work.setUpdateTime(vm.getUpdated());

server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java

@@ -44,6 +44,7 @@ import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
@@ -420,7 +421,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
     }
 
     @Override
-    public UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params) {
+    public Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params) {
         logger.error("Unsupported operation: cannot clone external VM");
         return null;
     }
@@ -430,4 +431,16 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
         logger.error("Unsupported operation: cannot remove external VM");
         return false;
     }
+
+    @Override
+    public String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf) {
+        logger.error("Unsupported operation: cannot create template file");
+        return null;
+    }
+
+    @Override
+    public boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir) {
+        logger.error("Unsupported operation: cannot remove template file");
+        return false;
+    }
 }

server/src/main/java/com/cloud/network/NetworkModelImpl.java

@@ -2387,7 +2387,9 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi
                 throw new InvalidParameterValueException("endIPv6 is not in ip6cidr indicated network!");
             }
         }
+    }
 
+    public void checkIp6CidrSizeEqualTo64(String ip6Cidr) {
         int cidrSize = NetUtils.getIp6CidrSize(ip6Cidr);
         // we only support cidr == 64
         if (cidrSize != 64) {

server/src/main/java/com/cloud/network/NetworkServiceImpl.java

@@ -37,7 +37,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -1591,6 +1590,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
                 endIPv6 = startIPv6;
             }
             _networkModel.checkIp6Parameters(startIPv6, endIPv6, ip6Gateway, ip6Cidr);
+            if (!GuestType.Shared.equals(ntwkOff.getGuestType())) {
+                _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr);
+            }
 
             if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() != Network.GuestType.Shared) {
                 throw new InvalidParameterValueException("Can only support create IPv6 network with advance shared network!");
@@ -2278,6 +2280,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         Long associatedNetworkId = cmd.getAssociatedNetworkId();
         String networkFilterStr = cmd.getNetworkFilter();
 
+        boolean applyManualPagination = CollectionUtils.isNotEmpty(supportedServicesStr) ||
+                Boolean.TRUE.equals(canUseForDeploy);
+
         String vlanId = null;
         if (cmd instanceof ListNetworksCmdByAdmin) {
             vlanId = ((ListNetworksCmdByAdmin)cmd).getVlan();
@@ -2363,7 +2368,13 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             isRecursive = true;
         }
 
-        Filter searchFilter = new Filter(NetworkVO.class, "id", false, null, null);
+        Long offset = cmd.getStartIndex();
+        Long limit = cmd.getPageSizeVal();
+        if (applyManualPagination) {
+            offset = null;
+            limit = null;
+        }
+        Filter searchFilter = new Filter(NetworkVO.class, "id", false, offset, limit);
         SearchBuilder<NetworkVO> sb = _networksDao.createSearchBuilder();
 
         if (forVpc != null) {
@@ -2418,112 +2429,122 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             sb.join("associatedNetworkSearch", associatedNetworkSearch, sb.entity().getId(), associatedNetworkSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER);
         }
 
-        List<NetworkVO> networksToReturn = new ArrayList<NetworkVO>();
+        SearchCriteria<NetworkVO> mainSearchCriteria = createNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId,
+                guestIpType, trafficType, physicalNetworkId, networkOfferingId, aclType, restartRequired,
+                specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId);
+        SearchCriteria<NetworkVO> additionalSearchCriteria = _networksDao.createSearchCriteria();
 
         if (isSystem == null || !isSystem) {
             if (!permittedAccounts.isEmpty()) {
                 if (Arrays.asList(Network.NetworkFilter.Account, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
                     //get account level networks
-                    networksToReturn.addAll(listAccountSpecificNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC,
-                            aclType, skipProjectNetworks, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, permittedAccounts));
+                            getAccountSpecificNetworksSearchCriteria(sb, permittedAccounts, skipProjectNetworks));
                 }
                 if (domainId != null && Arrays.asList(Network.NetworkFilter.Domain, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
                     //get domain level networks
-                    networksToReturn.addAll(listDomainLevelNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    SearchCriteria<NetworkVO> domainLevelSC = getDomainLevelNetworksSearchCriteria(sb, domainId, false);
-                            aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, domainId, false));
+                    if (domainLevelSC != null) {
+                        additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC, domainLevelSC);
+                    }
                 }
                 if (Arrays.asList(Network.NetworkFilter.Shared, Network.NetworkFilter.All).contains(networkFilter)) {
                     // get shared networks
-                    List<NetworkVO> sharedNetworks = listSharedNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    SearchCriteria<NetworkVO> sharedNetworksSC = getSharedNetworksSearchCriteria(sb, permittedAccounts);
-                            aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, permittedAccounts);
+                    if (sharedNetworksSC != null) {
-                    addNetworksToReturnIfNotExist(networksToReturn, sharedNetworks);
+                        additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC, sharedNetworksSC);
-
+                    }
                 }
             } else {
                 if (Arrays.asList(Network.NetworkFilter.Account, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
                     //add account specific networks
-                    networksToReturn.addAll(listAccountSpecificNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC,
-                            aclType, skipProjectNetworks, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive));
+                            getAccountSpecificNetworksByDomainPathSearchCriteria(sb, path, isRecursive,
+                                    skipProjectNetworks));
                 }
                 if (Arrays.asList(Network.NetworkFilter.Domain, Network.NetworkFilter.AccountDomain, Network.NetworkFilter.All).contains(networkFilter)) {
                     //add domain specific networks of domain + parent domains
-                    networksToReturn.addAll(listDomainSpecificNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    SearchCriteria<NetworkVO> domainSpecificNetworksByDomainPathSC =
-                            aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive));
+                            getDomainSpecificNetworksByDomainPathSearchCriteria(sb, path, isRecursive);
+                    if (domainSpecificNetworksByDomainPathSC != null) {
+                        additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC, domainSpecificNetworksByDomainPathSC);
+                    }
                     //add networks of subdomains
                     if (domainId == null) {
-                        networksToReturn.addAll(listDomainLevelNetworks(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                        SearchCriteria<NetworkVO> domainLevelSC = getDomainLevelNetworksSearchCriteria(sb, caller.getDomainId(), true);
-                            aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, caller.getDomainId(), true));
+                        if (domainLevelSC != null) {
+                            additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC, domainLevelSC);
+                        }
                     }
                 }
                 if (Arrays.asList(Network.NetworkFilter.Shared, Network.NetworkFilter.All).contains(networkFilter)) {
                     // get shared networks
-                    List<NetworkVO> sharedNetworks = listSharedNetworksByDomainPath(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+                    SearchCriteria<NetworkVO> sharedNetworksSC = getSharedNetworksByDomainPathSearchCriteria(sb, path, isRecursive);
-                            aclType, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter, path, isRecursive);
+                    if (sharedNetworksSC != null) {
-                    addNetworksToReturnIfNotExist(networksToReturn, sharedNetworks);
+                        additionalSearchCriteria.addOr("id", SearchCriteria.Op.SC, sharedNetworksSC);
                     }
                 }
+            }
+            if (CollectionUtils.isNotEmpty(additionalSearchCriteria.getValues())) {
+                mainSearchCriteria.addAnd("id", SearchCriteria.Op.SC, additionalSearchCriteria);
+            }
         } else {
-            networksToReturn = _networksDao.search(buildNetworkSearchCriteria(sb, keyword, id, isSystem, zoneId, guestIpType, trafficType, physicalNetworkId, networkOfferingId,
+            if (skipProjectNetworks) {
-                    null, true, restartRequired, specifyIpRanges, vpcId, tags, display, vlanId, associatedNetworkId), searchFilter);
+                mainSearchCriteria.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
+            } else {
+                mainSearchCriteria.setJoinParameters("accountSearch", "typeEQ", Account.Type.PROJECT);
             }
+        }
+        Pair<List<NetworkVO>, Integer> result = _networksDao.searchAndCount(mainSearchCriteria, searchFilter);
+        List<NetworkVO> networksToReturn = result.first();
 
         if (supportedServicesStr != null && !supportedServicesStr.isEmpty() && !networksToReturn.isEmpty()) {
-            List<NetworkVO> supportedNetworks = new ArrayList<NetworkVO>();
+            List<NetworkVO> supportedNetworks = new ArrayList<>();
-            Service[] suppportedServices = new Service[supportedServicesStr.size()];
+            Service[] supportedServices = new Service[supportedServicesStr.size()];
             int i = 0;
             for (String supportedServiceStr : supportedServicesStr) {
                 Service service = Service.getService(supportedServiceStr);
                 if (service == null) {
                     throw new InvalidParameterValueException("Invalid service specified " + supportedServiceStr);
                 } else {
-                    suppportedServices[i] = service;
+                    supportedServices[i] = service;
                 }
                 i++;
             }
-
             for (NetworkVO network : networksToReturn) {
-                if (areServicesSupportedInNetwork(network.getId(), suppportedServices)) {
+                if (areServicesSupportedInNetwork(network.getId(), supportedServices)) {
                     supportedNetworks.add(network);
                 }
             }
-
             networksToReturn = supportedNetworks;
         }
 
         if (canUseForDeploy != null) {
-            List<NetworkVO> networksForDeploy = new ArrayList<NetworkVO>();
+            List<NetworkVO> networksForDeploy = new ArrayList<>();
             for (NetworkVO network : networksToReturn) {
                 if (_networkModel.canUseForDeploy(network) == canUseForDeploy) {
                     networksForDeploy.add(network);
                 }
             }
-
             networksToReturn = networksForDeploy;
         }
 
+        if (applyManualPagination) {
             //Now apply pagination
             List<? extends Network> wPagination = com.cloud.utils.StringUtils.applyPagination(networksToReturn, cmd.getStartIndex(), cmd.getPageSizeVal());
             if (wPagination != null) {
-            Pair<List<? extends Network>, Integer> listWPagination = new Pair<List<? extends Network>, Integer>(wPagination, networksToReturn.size());
+                Pair<List<? extends Network>, Integer> listWPagination = new Pair<>(wPagination, networksToReturn.size());
                 return listWPagination;
             }
-
+            return new Pair<>(networksToReturn, networksToReturn.size());
-        return new Pair<List<? extends Network>, Integer>(networksToReturn, networksToReturn.size());
         }
 
-    private void addNetworksToReturnIfNotExist(final List<NetworkVO> networksToReturn, final List<NetworkVO> sharedNetworks) {
+        return new Pair<>(result.first(), result.second());
-        Set<Long> networkIds = networksToReturn.stream()
-                .map(NetworkVO::getId)
-                .collect(Collectors.toSet());
-        List<NetworkVO> sharedNetworksToReturn = sharedNetworks.stream()
-                .filter(network -> ! networkIds.contains(network.getId()))
-                .collect(Collectors.toList());
-        networksToReturn.addAll(sharedNetworksToReturn);
     }
 
-    private SearchCriteria<NetworkVO> buildNetworkSearchCriteria(SearchBuilder<NetworkVO> sb, String keyword, Long id,
+    private SearchCriteria<NetworkVO> createNetworkSearchCriteria(SearchBuilder<NetworkVO> sb, String keyword, Long id,
                                                                  Boolean isSystem, Long zoneId, String guestIpType, String trafficType, Long physicalNetworkId,
-            Long networkOfferingId, String aclType, boolean skipProjectNetworks, Boolean restartRequired,
+                                                                 Long networkOfferingId, String aclType, Boolean restartRequired,
                                                                  Boolean specifyIpRanges, Long vpcId, Map<String, String> tags, Boolean display, String vlanId, Long associatedNetworkId) {
 
         SearchCriteria<NetworkVO> sc = sb.create();
@@ -2566,12 +2587,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             sc.addAnd("physicalNetworkId", SearchCriteria.Op.EQ, physicalNetworkId);
         }
 
-        if (skipProjectNetworks) {
-            sc.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
-        } else {
-            sc.setJoinParameters("accountSearch", "typeEQ", Account.Type.PROJECT);
-        }
-
         if (restartRequired != null) {
             sc.addAnd("restartRequired", SearchCriteria.Op.EQ, restartRequired);
         }
@@ -2612,8 +2627,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         return sc;
     }
 
-    private List<NetworkVO> listDomainLevelNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, long domainId, boolean parentDomainsOnly) {
+    private SearchCriteria<NetworkVO> getDomainLevelNetworksSearchCriteria(SearchBuilder<NetworkVO> sb, long domainId, boolean parentDomainsOnly) {
-        List<Long> networkIds = new ArrayList<Long>();
+        List<Long> networkIds = new ArrayList<>();
         Set<Long> allowedDomains = _domainMgr.getDomainParentIds(domainId);
         List<NetworkDomainVO> maps = _networkDomainDao.listDomainNetworkMapByDomain(allowedDomains.toArray());
 
@@ -2628,48 +2643,55 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         }
 
         if (!networkIds.isEmpty()) {
-            SearchCriteria<NetworkVO> domainSC = _networksDao.createSearchCriteria();
+            SearchCriteria<NetworkVO> domainSC = sb.create();
+            domainSC.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
             domainSC.addAnd("id", SearchCriteria.Op.IN, networkIds.toArray());
             domainSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Domain.toString());
+            return domainSC;
+        }
+        return null;
+    }
 
-            sc.addAnd("id", SearchCriteria.Op.SC, domainSC);
+    private SearchCriteria<NetworkVO> getAccountSpecificNetworksSearchCriteria(SearchBuilder<NetworkVO> sb,
-            return _networksDao.search(sc, searchFilter);
+           List<Long> permittedAccounts, boolean skipProjectNetworks) {
+        SearchCriteria<NetworkVO> accountSC = sb.create();
+        if (skipProjectNetworks) {
+            accountSC.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
         } else {
-            return new ArrayList<NetworkVO>();
+            accountSC.setJoinParameters("accountSearch", "typeEQ", Account.Type.PROJECT);
         }
-    }
-
-    private List<NetworkVO> listAccountSpecificNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, List<Long> permittedAccounts) {
-        SearchCriteria<NetworkVO> accountSC = _networksDao.createSearchCriteria();
         if (!permittedAccounts.isEmpty()) {
             accountSC.addAnd("accountId", SearchCriteria.Op.IN, permittedAccounts.toArray());
         }
-
         accountSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Account.toString());
-
+        return accountSC;
-        sc.addAnd("id", SearchCriteria.Op.SC, accountSC);
-        return _networksDao.search(sc, searchFilter);
     }
 
-    private List<NetworkVO> listAccountSpecificNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
+    private SearchCriteria<NetworkVO> getAccountSpecificNetworksByDomainPathSearchCriteria(SearchBuilder<NetworkVO> sb,
-        SearchCriteria<NetworkVO> accountSC = _networksDao.createSearchCriteria();
+           String path, boolean isRecursive, boolean skipProjectNetworks) {
+        SearchCriteria<NetworkVO> accountSC = sb.create();
+        if (skipProjectNetworks) {
+            accountSC.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
+        } else {
+            accountSC.setJoinParameters("accountSearch", "typeEQ", Account.Type.PROJECT);
+        }
         accountSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Account.toString());
 
         if (path != null) {
             if (isRecursive) {
-                sc.setJoinParameters("domainSearch", "path", path + "%");
+                accountSC.setJoinParameters("domainSearch", "path", path + "%");
             } else {
-                sc.setJoinParameters("domainSearch", "path", path);
+                accountSC.setJoinParameters("domainSearch", "path", path);
             }
         }
 
-        sc.addAnd("id", SearchCriteria.Op.SC, accountSC);
+        return accountSC;
-        return _networksDao.search(sc, searchFilter);
     }
 
-    private List<NetworkVO> listDomainSpecificNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
+    private SearchCriteria<NetworkVO> getDomainSpecificNetworksByDomainPathSearchCriteria(SearchBuilder<NetworkVO> sb,
+            String path, boolean isRecursive) {
 
-        Set<Long> allowedDomains = new HashSet<Long>();
+        Set<Long> allowedDomains = new HashSet<>();
         if (path != null) {
             if (isRecursive) {
                 allowedDomains = _domainMgr.getDomainChildrenIds(path);
@@ -2679,7 +2701,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             }
         }
 
-        List<Long> networkIds = new ArrayList<Long>();
+        List<Long> networkIds = new ArrayList<>();
 
         List<NetworkDomainVO> maps = _networkDomainDao.listDomainNetworkMapByDomain(allowedDomains.toArray());
 
@@ -2688,30 +2710,28 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         }
 
         if (!networkIds.isEmpty()) {
-            SearchCriteria<NetworkVO> domainSC = _networksDao.createSearchCriteria();
+            SearchCriteria<NetworkVO> domainSC = sb.create();
+            domainSC.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
             domainSC.addAnd("id", SearchCriteria.Op.IN, networkIds.toArray());
             domainSC.addAnd("aclType", SearchCriteria.Op.EQ, ACLType.Domain.toString());
-
+            return domainSC;
-            sc.addAnd("id", SearchCriteria.Op.SC, domainSC);
-            return _networksDao.search(sc, searchFilter);
-        } else {
-            return new ArrayList<NetworkVO>();
         }
+        return null;
     }
 
-    private List<NetworkVO> listSharedNetworks(SearchCriteria<NetworkVO> sc, Filter searchFilter, List<Long> permittedAccounts) {
+    private SearchCriteria<NetworkVO> getSharedNetworksSearchCriteria(SearchBuilder<NetworkVO> sb, List<Long> permittedAccounts) {
         List<Long> sharedNetworkIds = _networkPermissionDao.listPermittedNetworkIdsByAccounts(permittedAccounts);
         if (!sharedNetworkIds.isEmpty()) {
-            SearchCriteria<NetworkVO> ssc = _networksDao.createSearchCriteria();
+            SearchCriteria<NetworkVO> ssc = sb.create();
+            ssc.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
             ssc.addAnd("id", SearchCriteria.Op.IN, sharedNetworkIds.toArray());
-            sc.addAnd("id", SearchCriteria.Op.SC, ssc);
+            return ssc;
-            return _networksDao.search(sc, searchFilter);
         }
-        return new ArrayList<NetworkVO>();
+        return null;
     }
 
-    private List<NetworkVO> listSharedNetworksByDomainPath(SearchCriteria<NetworkVO> sc, Filter searchFilter, String path, boolean isRecursive) {
+    private SearchCriteria<NetworkVO> getSharedNetworksByDomainPathSearchCriteria(SearchBuilder<NetworkVO> sb, String path, boolean isRecursive) {
-        Set<Long> allowedDomains = new HashSet<Long>();
+        Set<Long> allowedDomains = new HashSet<>();
         if (path != null) {
             if (isRecursive) {
                 allowedDomains = _domainMgr.getDomainChildrenIds(path);
@@ -2733,13 +2753,13 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
 
             List<Long> sharedNetworkIds = _networkPermissionDao.listPermittedNetworkIdsByAccounts(allowedAccountsList);
             if (!sharedNetworkIds.isEmpty()) {
-                SearchCriteria<NetworkVO> ssc = _networksDao.createSearchCriteria();
+                SearchCriteria<NetworkVO> ssc = sb.create();
+                ssc.setJoinParameters("accountSearch", "typeNEQ", Account.Type.PROJECT);
                 ssc.addAnd("id", SearchCriteria.Op.IN, sharedNetworkIds.toArray());
-                sc.addAnd("id", SearchCriteria.Op.SC, ssc);
+                return ssc;
-                return _networksDao.search(sc, searchFilter);
             }
         }
-        return new ArrayList<NetworkVO>();
+        return null;
     }
 
     @Override

server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java

@@ -444,6 +444,7 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur
                         } else {
                             guestIp = _ipAddrMgr.acquireGuestIpAddress(network, nic.getRequestedIPv4());
                         }
+                        nic.setIpv4AllocationRaceCheck(true);
                     }
                     if (guestIp == null && network.getGuestType() != GuestType.L2 && !_networkModel.listNetworkOfferingServices(network.getNetworkOfferingId()).isEmpty()) {
                         throw new InsufficientVirtualNetworkCapacityException("Unable to acquire Guest IP" + " address for network " + network, DataCenter.class,

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -122,6 +122,7 @@ import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.HostPodDao;
 import com.cloud.dc.dao.VlanDao;
 import com.cloud.deploy.DeployDestination;
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.domain.Domain;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.ActionEventUtils;
@@ -2439,7 +2440,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
     protected void finalizeNetworkRulesForNetwork(final Commands cmds, final DomainRouterVO router, final Provider provider, final Long guestNetworkId) {
         logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start");
 
-        final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(router, provider, guestNetworkId);
+        final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(provider, guestNetworkId);
         final List<FirewallRule> firewallRulesEgress = new ArrayList<FirewallRule>();
         final List<FirewallRule> ipv6firewallRules = new ArrayList<>();
 
@@ -2646,7 +2647,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
     protected void finalizeIpAssocForNetwork(final Commands cmds, final VirtualRouter router, final Provider provider, final Long guestNetworkId,
             final Map<String, String> vlanMacAddress) {
 
-        final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(router, provider, guestNetworkId);
+        final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(provider, guestNetworkId);
 
         if (publicIps != null && !publicIps.isEmpty()) {
             logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start.");
@@ -2657,18 +2658,10 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         }
     }
 
-    protected ArrayList<? extends PublicIpAddress> getPublicIpsToApply(final VirtualRouter router, final Provider provider, final Long guestNetworkId,
+    protected ArrayList<? extends PublicIpAddress> getPublicIpsToApply(final Provider provider, final Long guestNetworkId,
             final com.cloud.network.IpAddress.State... skipInStates) {
-        final long ownerId = router.getAccountId();
-        final List<? extends IpAddress> userIps;
 
-        final Network guestNetwork = _networkDao.findById(guestNetworkId);
+        final List<? extends IpAddress> userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null);
-        if (guestNetwork.getGuestType() == GuestType.Shared) {
-            // ignore the account id for the shared network
-            userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null);
-        } else {
-            userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null);
-        }
 
         final List<PublicIp> allPublicIps = new ArrayList<PublicIp>();
         if (userIps != null && !userIps.isEmpty()) {
@@ -3008,6 +3001,14 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         return virtualRouter;
     }
 
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "restarting router VM for HA", async = true)
+    public void startRouterForHA(VirtualMachine vm, Map<Param, Object> params, DeploymentPlanner planner)
+            throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
+            OperationTimedoutException {
+        _itMgr.advanceStart(vm.getUuid(), params, planner);
+    }
+
     @Override
     public List<VirtualRouter> getRoutersForNetwork(final long networkId) {
         final List<DomainRouterVO> routers = _routerDao.findByNetwork(networkId);

server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java

@@ -757,7 +757,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
         }
 
         if (domainRouterVO.getState() == State.Starting || domainRouterVO.getState() == State.Running) {
-            final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(domainRouterVO, provider, guestNetworkId, IpAddress.State.Releasing);
+            final ArrayList<? extends PublicIpAddress> publicIps = getPublicIpsToApply(provider, guestNetworkId, IpAddress.State.Releasing);
 
             if (publicIps != null && !publicIps.isEmpty()) {
                 logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + domainRouterVO + " start.");

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -3371,6 +3371,15 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         return sc.list();
     }
 
+    @Override
+    public List<HostVO> listAllUpHostsInOneZoneByHypervisor(final HypervisorType type, final long dcId) {
+        final QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+        sc.and(sc.entity().getHypervisorType(), Op.EQ, type);
+        sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId);
+        sc.and(sc.entity().getStatus(), Op.EQ, Status.Up);
+        return sc.list();
+    }
+
     @Override
     public List<HostVO> listAllUpAndEnabledHostsInOneZone(final long dcId) {
         final QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.server;
 
+import static com.cloud.configuration.ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE;
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 
 import java.lang.management.ManagementFactory;
@@ -282,11 +283,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     protected static ConfigKey<Boolean> vmStatsCollectUserVMOnly = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.user.vm.only", "false",
             "When set to 'false' stats for system VMs will be collected otherwise stats collection will be done only for user VMs", true);
 
-    protected static ConfigKey<Long> vmStatsRemoveBatchSize = new ConfigKey<>("Advanced", Long.class, "vm.stats.remove.batch.size", "0", "Indicates the" +
-            " limit applied to delete vm_stats entries while running the clean-up task. With this, ACS will run the delete query, applying the limit, as many times as necessary" +
-            " to delete all entries older than the value defined in vm.stats.max.retention.time. This is advised when retaining several days of records, which can lead to slowness" +
-            " on the delete query. Zero (0) means that no limit will be applied, therefore, the query will run once and without limit, keeping the default behavior.", true);
-
     protected static ConfigKey<Boolean> vmDiskStatsRetentionEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.disk.stats.retention.enabled", "false",
             "When set to 'true' stats for VM disks will be stored in the database otherwise disk stats will not be stored", true);
 
@@ -1965,7 +1961,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
         logger.trace("Removing older VM stats records.");
         Date now = new Date();
         Date limit = DateUtils.addMinutes(now, -maxRetentionTime);
-        vmStatsDao.removeAllByTimestampLessThan(limit, vmStatsRemoveBatchSize.value());
+        vmStatsDao.removeAllByTimestampLessThan(limit, DELETE_QUERY_BATCH_SIZE.value());
     }
 
     /**
@@ -1984,7 +1980,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
         logger.trace("Removing older Volume stats records.");
         Date now = new Date();
         Date limit = DateUtils.addMinutes(now, -maxRetentionTime);
-        volumeStatsDao.removeAllByTimestampLessThan(limit);
+        volumeStatsDao.removeAllByTimestampLessThan(limit, DELETE_QUERY_BATCH_SIZE.value());
     }
 
     /**
@@ -2139,7 +2135,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri, vmStatsRemoveBatchSize,
+        return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri,
             vmStatsIncrementMetrics, vmStatsMaxRetentionTime, vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled, vmDiskStatsMaxRetentionTime,
                 MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL,
                 DATABASE_SERVER_STATUS_COLLECTION_INTERVAL,

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -410,6 +410,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     private final Map<String, HypervisorHostListener> hostListeners = new HashMap<>();
 
+    private static final String NFS_MOUNT_OPTIONS_INCORRECT = "An incorrect mount option was specified";
+
     public boolean share(VMInstanceVO vm, List<VolumeVO> vols, HostVO host, boolean cancelPreviousShare) throws StorageUnavailableException {
 
         // if pool is in maintenance and it is the ONLY pool available; reject
@@ -836,6 +838,53 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         return String.format("%s-%s-%s", StringUtils.trim(host.getName()), "local", storagePoolInformation.getUuid().split("-")[0]);
     }
 
+    protected void checkNfsMountOptions(String nfsMountOpts) throws InvalidParameterValueException {
+        String[] options = nfsMountOpts.replaceAll("\\s", "").split(",");
+        Map<String, String> optionsMap = new HashMap<>();
+        for (String option : options) {
+            String[] keyValue = option.split("=");
+            if (keyValue.length > 2) {
+                throw new InvalidParameterValueException("Invalid value for NFS option " + keyValue[0]);
+            }
+            if (optionsMap.containsKey(keyValue[0])) {
+                throw new InvalidParameterValueException("Duplicate NFS option values found for option " + keyValue[0]);
+            }
+            optionsMap.put(keyValue[0], null);
+        }
+    }
+
+    protected void checkNFSMountOptionsForCreate(Map<String, String> details, HypervisorType hypervisorType, String scheme) throws InvalidParameterValueException {
+        if (!details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) {
+            return;
+        }
+        if (!hypervisorType.equals(HypervisorType.KVM) && !hypervisorType.equals(HypervisorType.Simulator)) {
+            throw new InvalidParameterValueException("NFS options can not be set for the hypervisor type " + hypervisorType);
+        }
+        if (!"nfs".equals(scheme)) {
+            throw new InvalidParameterValueException("NFS options can only be set on pool type " + StoragePoolType.NetworkFilesystem);
+        }
+        checkNfsMountOptions(details.get(ApiConstants.NFS_MOUNT_OPTIONS));
+    }
+
+    protected void checkNFSMountOptionsForUpdate(Map<String, String> details, StoragePoolVO pool, Long accountId) throws InvalidParameterValueException {
+        if (!details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) {
+            return;
+        }
+        if (!_accountMgr.isRootAdmin(accountId)) {
+            throw new PermissionDeniedException("Only root admin can modify nfs options");
+        }
+        if (!pool.getHypervisor().equals(HypervisorType.KVM) && !pool.getHypervisor().equals((HypervisorType.Simulator))) {
+            throw new InvalidParameterValueException("NFS options can only be set for the hypervisor type " + HypervisorType.KVM);
+        }
+        if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+            throw new InvalidParameterValueException("NFS options can only be set on pool type " + StoragePoolType.NetworkFilesystem);
+        }
+        if (!pool.isInMaintenance()) {
+            throw new InvalidParameterValueException("The storage pool should be in maintenance mode to edit nfs options");
+        }
+        checkNfsMountOptions(details.get(ApiConstants.NFS_MOUNT_OPTIONS));
+    }
+
     @Override
     public PrimaryDataStoreInfo createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException {
         String providerName = cmd.getStorageProviderName();
@@ -900,6 +949,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
 
         Map<String, String> details = extractApiParamAsMap(cmd.getDetails());
+        checkNFSMountOptionsForCreate(details, hypervisorType, uriParams.get("scheme"));
+
         DataCenterVO zone = _dcDao.findById(cmd.getZoneId());
         if (zone == null) {
             throw new InvalidParameterValueException("unable to find zone by id " + zoneId);
@@ -1082,6 +1133,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
             throw new IllegalArgumentException("Unable to find storage pool with ID: " + id);
         }
 
+        Map<String, String> inputDetails = extractApiParamAsMap(cmd.getDetails());
+        checkNFSMountOptionsForUpdate(inputDetails, pool, cmd.getEntityOwnerId());
+
         String name = cmd.getName();
         if(StringUtils.isNotBlank(name)) {
             logger.debug("Updating Storage Pool name to: " + name);
@@ -1125,12 +1179,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
 
         // retrieve current details and merge/overlay input to capture changes
-        Map<String, String> inputDetails = extractApiParamAsMap(cmd.getDetails());
         Map<String, String> details = null;
-        if (inputDetails == null) {
-            details = _storagePoolDetailsDao.listDetailsKeyPairs(id);
-        } else {
         details = _storagePoolDetailsDao.listDetailsKeyPairs(id);
+        if (inputDetails != null) {
             details.putAll(inputDetails);
             changes = true;
         }
@@ -1229,6 +1280,32 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         return deleteDataStoreInternal(sPool, forced);
     }
 
+    @Override
+    public Pair<Map<String, String>, Boolean> getStoragePoolNFSMountOpts(StoragePool pool, Map<String, String> details) {
+        boolean details_added = false;
+        if (!pool.getPoolType().equals(Storage.StoragePoolType.NetworkFilesystem)) {
+            return new Pair<>(details, details_added);
+        }
+
+        StoragePoolDetailVO nfsMountOpts = _storagePoolDetailsDao.findDetail(pool.getId(), ApiConstants.NFS_MOUNT_OPTIONS);
+        if (nfsMountOpts != null) {
+            if (details == null) {
+                details = new HashMap<>();
+            }
+            details.put(ApiConstants.NFS_MOUNT_OPTIONS, nfsMountOpts.getValue());
+            details_added = true;
+        }
+        return new Pair<>(details, details_added);
+    }
+
+    public String getStoragePoolMountFailureReason(String reason) {
+        if (reason.toLowerCase().contains(NFS_MOUNT_OPTIONS_INCORRECT.toLowerCase())) {
+            return NFS_MOUNT_OPTIONS_INCORRECT;
+        } else {
+            return null;
+        }
+    }
+
     private boolean checkIfDataStoreClusterCanbeDeleted(StoragePoolVO sPool, boolean forced) {
         List<StoragePoolVO> childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId());
         boolean canDelete = true;
@@ -3796,7 +3873,6 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 MountDisabledStoragePool,
                 VmwareCreateCloneFull,
                 VmwareAllowParallelExecution,
-                ConvertVmwareInstanceToKvmTimeout,
                 DataStoreDownloadFollowRedirects
         };
     }

server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java

@@ -20,6 +20,7 @@ package com.cloud.storage;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import javax.inject.Inject;
 
@@ -28,6 +29,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -49,6 +51,7 @@ import com.cloud.storage.dao.VolumeDao;
 import com.cloud.user.Account;
 import com.cloud.user.User;
 import com.cloud.user.dao.UserDao;
+import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.ConsoleProxyVO;
 import com.cloud.vm.DomainRouterVO;
@@ -89,6 +92,8 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
     @Inject
     PrimaryDataStoreDao primaryDataStoreDao;
     @Inject
+    StoragePoolDetailsDao storagePoolDetailsDao;
+    @Inject
     DataStoreManager dataStoreMgr;
     @Inject
     protected ResourceManager _resourceMgr;
@@ -319,14 +324,25 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
         if (hosts == null || hosts.size() == 0) {
             return true;
         }
+
+        Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
         // add heartbeat
         for (HostVO host : hosts) {
-            ModifyStoragePoolCommand msPoolCmd = new ModifyStoragePoolCommand(true, pool);
+            ModifyStoragePoolCommand msPoolCmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
             final Answer answer = agentMgr.easySend(host.getId(), msPoolCmd);
             if (answer == null || !answer.getResult()) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("ModifyStoragePool add failed due to " + ((answer == null) ? "answer null" : answer.getDetails()));
                 }
+                if (answer != null && nfsMountOpts.second()) {
+                    logger.error(String.format("Unable to attach storage pool to the host %s due to %s",  host,  answer.getDetails()));
+                    StringBuilder exceptionSB = new StringBuilder("Unable to attach storage pool to the host ").append(host.getName());
+                    String reason = storageManager.getStoragePoolMountFailureReason(answer.getDetails());
+                    if (reason!= null) {
+                        exceptionSB.append(". ").append(reason).append(".");
+                    }
+                    throw new CloudRuntimeException(exceptionSB.toString());
+                }
             } else {
                 if (logger.isDebugEnabled()) {
                     logger.debug("ModifyStoragePool add succeeded");

server/src/main/java/com/cloud/storage/secondary/SecondaryStorageVmManager.java

@@ -17,16 +17,24 @@
 package com.cloud.storage.secondary;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StartupCommand;
+import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.host.HostVO;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Manager;
 import com.cloud.vm.SecondaryStorageVm;
 import com.cloud.vm.SecondaryStorageVmVO;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
 
 public interface SecondaryStorageVmManager extends Manager {
 
@@ -47,6 +55,10 @@ public interface SecondaryStorageVmManager extends Manager {
 
     public SecondaryStorageVmVO startSecStorageVm(long ssVmVmId);
 
+    void startSecStorageVmForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+            DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException;
+
     public boolean stopSecStorageVm(long ssVmVmId);
 
     public boolean rebootSecStorageVm(long ssVmVmId);

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -3266,6 +3266,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         _itMgr.advanceStart(vm.getUuid(), null, null);
     }
 
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_VM_START, eventDescription = "restarting VM for HA", async = true)
+    public void startVirtualMachineForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+           DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException {
+        _itMgr.advanceStart(vm.getUuid(), params, planner);
+    }
+
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_VM_REBOOT, eventDescription = "rebooting Vm", async = true)
     public UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ResourceAllocationException {
@@ -8672,10 +8680,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_STOP, vm.getAccountId(), vm.getDataCenterId(),
                     vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(),
                     vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm());
+
             resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering, template);
             resourceNotDecremented = false;
         }
-
         // VM destroy usage event
         UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterId(),
                 vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(),

server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java

@@ -464,7 +464,7 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ
             Map<Host, Boolean> requiresStorageMotion = hostsForMigrationOfVM.third();
 
             for (Host destHost : compatibleDestinationHosts) {
-                if (!suitableDestinationHosts.contains(destHost)) {
+                if (!suitableDestinationHosts.contains(destHost) || cluster.getId() != destHost.getClusterId()) {
                     continue;
                 }
                 Ternary<Double, Double, Double> metrics = algorithm.getMetrics(cluster.getId(), vm,

server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java

@@ -19,6 +19,8 @@ package org.apache.cloudstack.vm;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CheckConvertInstanceAnswer;
+import com.cloud.agent.api.CheckConvertInstanceCommand;
 import com.cloud.agent.api.CheckVolumeAnswer;
 import com.cloud.agent.api.CheckVolumeCommand;
 import com.cloud.agent.api.ConvertInstanceAnswer;
@@ -95,7 +97,6 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Storage;
-import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.VMTemplateVO;
@@ -689,7 +690,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         // Check for duplicate hostname in network, get all vms hostNames in the network
         List<String> hostNames = vmDao.listDistinctHostNames(network.getId());
         if (CollectionUtils.isNotEmpty(hostNames) && hostNames.contains(hostName)) {
-            throw new InvalidParameterValueException(String.format("VM with Name [%s] already exists in the network [%s] domain [%s]. Cannot import another VM with the same name. Pleasy try again with a different name.", hostName, network, network.getNetworkDomain()));
+            throw new InvalidParameterValueException(String.format("VM with Name [%s] already exists in the network [%s] domain [%s]. Cannot import another VM with the same name. Please try again with a different name.", hostName, network, network.getNetworkDomain()));
         }
     }
 
@@ -1564,17 +1565,30 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         return userVm;
     }
 
-    private UnmanagedInstanceTO cloneSourceVmwareUnmanagedInstance(String vcenter, String datacenterName, String username, String password, String clusterName, String sourceHostName, String sourceVM) {
+    private Pair<UnmanagedInstanceTO, Boolean> getSourceVmwareUnmanagedInstance(String vcenter, String datacenterName, String username,
+                                                                 String password, String clusterName, String sourceHostName,
+                                                                 String sourceVM) {
         HypervisorGuru vmwareGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.VMware);
 
         Map<String, String> params = createParamsForTemplateFromVmwareVmMigration(vcenter, datacenterName,
                 username, password, clusterName, sourceHostName, sourceVM);
 
-        return vmwareGuru.cloneHypervisorVMOutOfBand(sourceHostName, sourceVM, params);
+        return vmwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(sourceHostName, sourceVM, params);
+    }
+
+    private String createOvfTemplateOfSourceVmwareUnmanagedInstance(String vcenter, String datacenterName, String username,
+                                                                    String password, String clusterName, String sourceHostName,
+                                                                    String sourceVMwareInstanceName, DataStoreTO convertLocation, int threadsCountToExportOvf) {
+        HypervisorGuru vmwareGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.VMware);
+
+        Map<String, String> params = createParamsForTemplateFromVmwareVmMigration(vcenter, datacenterName,
+                username, password, clusterName, sourceHostName, sourceVMwareInstanceName);
+
+        return vmwareGuru.createVMTemplateOutOfBand(sourceHostName, sourceVMwareInstanceName, params, convertLocation, threadsCountToExportOvf);
     }
 
     protected UserVm importUnmanagedInstanceFromVmwareToKvm(DataCenter zone, Cluster destinationCluster, VMTemplateVO template,
-                                                          String sourceVM, String displayName, String hostName,
+                                                          String sourceVMName, String displayName, String hostName,
                                                           Account caller, Account owner, long userId,
                                                           ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap,
                                                           Map<String, Long> nicNetworkMap, Map<String, Network.IpAddresses> nicIpAddressMap,
@@ -1601,7 +1615,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         if (existingVcenterId != null) {
             VmwareDatacenterVO existingDC = vmwareDatacenterDao.findById(existingVcenterId);
             if (existingDC == null) {
-                String err = String.format("Cannot find any existing Vmware DC with ID %s", existingVcenterId);
+                String err = String.format("Cannot find any existing VMware DC with ID %s", existingVcenterId);
                 logger.error(err);
                 throw new CloudRuntimeException(err);
             }
@@ -1611,21 +1625,52 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
             password = existingDC.getPassword();
         }
 
-        UnmanagedInstanceTO clonedInstance = null;
+        boolean isClonedInstance = false;
+        UnmanagedInstanceTO sourceVMwareInstance = null;
+        DataStoreTO temporaryConvertLocation = null;
+        String ovfTemplateOnConvertLocation = null;
         try {
+            HostVO convertHost = selectInstanceConversionKVMHostInCluster(destinationCluster, convertInstanceHostId);
+            CheckConvertInstanceAnswer conversionSupportAnswer = checkConversionSupportOnHost(convertHost, sourceVMName, false);
+            logger.debug(String.format("The host %s (%s) is selected to execute the conversion of the instance %s" +
+                    " from VMware to KVM ", convertHost.getId(), convertHost.getName(), sourceVMName));
+
+            temporaryConvertLocation = selectInstanceConversionTemporaryLocation(destinationCluster, convertStoragePoolId);
+            List<StoragePoolVO> convertStoragePools = findInstanceConversionStoragePoolsInCluster(destinationCluster);
+            long importStartTime = System.currentTimeMillis();
+            Pair<UnmanagedInstanceTO, Boolean> sourceInstanceDetails = getSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password, clusterName, sourceHostName, sourceVMName);
+            sourceVMwareInstance = sourceInstanceDetails.first();
+            isClonedInstance = sourceInstanceDetails.second();
+            boolean isWindowsVm = sourceVMwareInstance.getOperatingSystem().toLowerCase().contains("windows");
+            if (isWindowsVm) {
+                checkConversionSupportOnHost(convertHost, sourceVMName, true);
+            }
+
             String instanceName = getGeneratedInstanceName(owner);
-            clonedInstance = cloneSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password,
+            checkNetworkingBeforeConvertingVmwareInstance(zone, owner, instanceName, hostName, sourceVMwareInstance, nicNetworkMap, nicIpAddressMap, forced);
-                    clusterName, sourceHostName, sourceVM);
+            UnmanagedInstanceTO convertedInstance;
-            checkNetworkingBeforeConvertingVmwareInstance(zone, owner, instanceName, hostName, clonedInstance, nicNetworkMap, nicIpAddressMap, forced);
+            if (cmd.getForceMsToImportVmFiles() || !conversionSupportAnswer.isOvfExportSupported()) {
-            UnmanagedInstanceTO convertedInstance = convertVmwareInstanceToKVM(vcenter, datacenterName, clusterName, username, password,
+                // Uses MS for OVF export to temporary conversion location
-                    sourceHostName, clonedInstance, destinationCluster, convertInstanceHostId, convertStoragePoolId);
+                int noOfThreads = UnmanagedVMsManager.ThreadsOnMSToImportVMwareVMFiles.value();
-            sanitizeConvertedInstance(convertedInstance, clonedInstance);
+                ovfTemplateOnConvertLocation = createOvfTemplateOfSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password,
+                        clusterName, sourceHostName, sourceVMwareInstance.getName(), temporaryConvertLocation, noOfThreads);
+                convertedInstance = convertVmwareInstanceToKVMWithOVFOnConvertLocation(sourceVMName, sourceVMwareInstance, convertHost, convertStoragePools,
+                        temporaryConvertLocation, ovfTemplateOnConvertLocation);
+            } else {
+                // Uses KVM Host for OVF export to temporary conversion location, through ovftool
+                convertedInstance = convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(sourceVMName, sourceVMwareInstance, convertHost, convertStoragePools,
+                        temporaryConvertLocation, vcenter, username, password, datacenterName);
+            }
+
+            sanitizeConvertedInstance(convertedInstance, sourceVMwareInstance);
             UserVm userVm = importVirtualMachineInternal(convertedInstance, instanceName, zone, destinationCluster, null,
                     template, displayName, hostName, caller, owner, userId,
                     serviceOffering, dataDiskOfferingMap,
                     nicNetworkMap, nicIpAddressMap,
                     details, false, forced, false);
-            logger.debug(String.format("VM %s imported successfully", sourceVM));
+            long timeElapsedInSecs = (System.currentTimeMillis() - importStartTime) / 1000;
+            logger.debug(String.format("VMware VM %s imported successfully to CloudStack instance %s (%s), Time taken: %d secs, OVF files imported from %s, Source VMware VM details - OS: %s, PowerState: %s, Disks: %s, NICs: %s",
+                    sourceVMName, instanceName, displayName, timeElapsedInSecs, (ovfTemplateOnConvertLocation != null)? "MS" : "KVM Host", sourceVMwareInstance.getOperatingSystem(), sourceVMwareInstance.getPowerState(), sourceVMwareInstance.getDisks(), sourceVMwareInstance.getNics()));
             return userVm;
         } catch (CloudRuntimeException e) {
             logger.error(String.format("Error importing VM: %s", e.getMessage()), e);
@@ -1633,20 +1678,25 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
                     cmd.getEventDescription(), null, null, 0);
             throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
         } finally {
-            removeClonedInstance(vcenter, datacenterName, username, password, sourceHostName, clonedInstance.getName(), sourceVM);
+            if (isClonedInstance && sourceVMwareInstance != null) {
+                removeClonedInstance(vcenter, datacenterName, username, password, sourceHostName, sourceVMwareInstance.getName(), sourceVMName);
+            }
+            if (temporaryConvertLocation != null  && StringUtils.isNotBlank(ovfTemplateOnConvertLocation)) {
+                removeTemplate(temporaryConvertLocation, ovfTemplateOnConvertLocation);
+            }
         }
     }
 
     private void checkNetworkingBeforeConvertingVmwareInstance(DataCenter zone, Account owner, String instanceName,
-                                                               String hostName, UnmanagedInstanceTO clonedInstance,
+                                                               String hostName, UnmanagedInstanceTO sourceVMwareInstance,
                                                                Map<String, Long> nicNetworkMap,
                                                                Map<String, Network.IpAddresses> nicIpAddressMap,
                                                                boolean forced) {
-        List<UnmanagedInstanceTO.Nic> nics = clonedInstance.getNics();
+        List<UnmanagedInstanceTO.Nic> nics = sourceVMwareInstance.getNics();
         List<Long> networkIds = new ArrayList<>(nicNetworkMap.values());
         if (nics.size() != networkIds.size()) {
             String msg = String.format("Different number of nics found on instance %s: %s vs %s nics provided",
-                    clonedInstance.getName(), nics.size(), networkIds.size());
+                    sourceVMwareInstance.getName(), nics.size(), networkIds.size());
             logger.error(msg);
             throw new CloudRuntimeException(msg);
         }
@@ -1674,8 +1724,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
     private void checkUnmanagedNicAndNetworkMacAddressForImport(NetworkVO network, UnmanagedInstanceTO.Nic nic, boolean forced) {
         NicVO existingNic = nicDao.findByNetworkIdAndMacAddress(network.getId(), nic.getMacAddress());
         if (existingNic != null && !forced) {
-            String err = String.format("NIC with MAC address = %s exists on network with ID = %s and forced flag is disabled",
+            String err = String.format("NIC with MAC address %s already exists on network with ID %s and forced flag is disabled. " +
-                    nic.getMacAddress(), network.getId());
+                    "Retry with forced flag enabled if a new MAC address to be generated.", nic.getMacAddress(), network.getUuid());
             logger.error(err);
             throw new CloudRuntimeException(err);
         }
@@ -1690,38 +1740,44 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         return VirtualMachineName.getVmName(id, owner.getId(), instanceSuffix);
     }
 
-    private void sanitizeConvertedInstance(UnmanagedInstanceTO convertedInstance, UnmanagedInstanceTO clonedInstance) {
+    private void sanitizeConvertedInstance(UnmanagedInstanceTO convertedInstance, UnmanagedInstanceTO sourceVMwareInstance) {
-        convertedInstance.setCpuCores(clonedInstance.getCpuCores());
+        convertedInstance.setCpuCores(sourceVMwareInstance.getCpuCores());
-        convertedInstance.setCpuSpeed(clonedInstance.getCpuSpeed());
+        convertedInstance.setCpuSpeed(sourceVMwareInstance.getCpuSpeed());
-        convertedInstance.setCpuCoresPerSocket(clonedInstance.getCpuCoresPerSocket());
+        convertedInstance.setCpuCoresPerSocket(sourceVMwareInstance.getCpuCoresPerSocket());
-        convertedInstance.setMemory(clonedInstance.getMemory());
+        convertedInstance.setMemory(sourceVMwareInstance.getMemory());
         convertedInstance.setPowerState(UnmanagedInstanceTO.PowerState.PowerOff);
         List<UnmanagedInstanceTO.Disk> convertedInstanceDisks = convertedInstance.getDisks();
-        List<UnmanagedInstanceTO.Disk> clonedInstanceDisks = clonedInstance.getDisks();
+        List<UnmanagedInstanceTO.Disk> sourceVMwareInstanceDisks = sourceVMwareInstance.getDisks();
         for (int i = 0; i < convertedInstanceDisks.size(); i++) {
             UnmanagedInstanceTO.Disk disk = convertedInstanceDisks.get(i);
-            disk.setDiskId(clonedInstanceDisks.get(i).getDiskId());
+            disk.setDiskId(sourceVMwareInstanceDisks.get(i).getDiskId());
         }
         List<UnmanagedInstanceTO.Nic> convertedInstanceNics = convertedInstance.getNics();
-        List<UnmanagedInstanceTO.Nic> clonedInstanceNics = clonedInstance.getNics();
+        List<UnmanagedInstanceTO.Nic> sourceVMwareInstanceNics = sourceVMwareInstance.getNics();
-        if (CollectionUtils.isEmpty(convertedInstanceNics) && CollectionUtils.isNotEmpty(clonedInstanceNics)) {
+        if (CollectionUtils.isEmpty(convertedInstanceNics) && CollectionUtils.isNotEmpty(sourceVMwareInstanceNics)) {
-            for (UnmanagedInstanceTO.Nic nic : clonedInstanceNics) {
+            for (UnmanagedInstanceTO.Nic nic : sourceVMwareInstanceNics) {
                 // In case the NICs information is not parsed from the converted XML domain, use the cloned instance NICs with virtio adapter
                 nic.setAdapterType("virtio");
             }
-            convertedInstance.setNics(clonedInstanceNics);
+            convertedInstance.setNics(sourceVMwareInstanceNics);
-        } else {
             for (int i = 0; i < convertedInstanceNics.size(); i++) {
                 UnmanagedInstanceTO.Nic nic = convertedInstanceNics.get(i);
-                nic.setNicId(clonedInstanceNics.get(i).getNicId());
+                nic.setNicId(sourceVMwareInstanceNics.get(i).getNicId());
+            }
+        } else if (CollectionUtils.isNotEmpty(convertedInstanceNics) && CollectionUtils.isNotEmpty(sourceVMwareInstanceNics)
+                && convertedInstanceNics.size() == sourceVMwareInstanceNics.size()) {
+            for (int i = 0; i < convertedInstanceNics.size(); i++) {
+                UnmanagedInstanceTO.Nic nic = convertedInstanceNics.get(i);
+                nic.setNicId(sourceVMwareInstanceNics.get(i).getNicId());
+                if (nic.getMacAddress() == null) {
+                    nic.setMacAddress(sourceVMwareInstanceNics.get(i).getMacAddress());
+                }
             }
         }
     }
 
-    private void removeClonedInstance(String vcenter, String datacenterName,
+    private void removeClonedInstance(String vcenter, String datacenterName, String username, String password,
-                                      String username, String password,
+                                      String sourceHostName, String clonedInstanceName, String sourceVM) {
-                                      String sourceHostName, String clonedInstanceName,
-                                      String sourceVM) {
         HypervisorGuru vmwareGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.VMware);
         Map<String, String> params = createParamsForRemoveClonedInstance(vcenter, datacenterName, username, password, sourceVM);
         boolean result = vmwareGuru.removeClonedHypervisorVMOutOfBand(sourceHostName, clonedInstanceName, params);
@@ -1731,10 +1787,23 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
             logger.warn(msg);
             return;
         }
-        logger.debug(String.format("Removed the cloned instance %s from VMWare datacenter %s:%s",
+        logger.debug(String.format("Removed the cloned instance %s from VMWare datacenter %s/%s",
                 clonedInstanceName, vcenter, datacenterName));
     }
 
+    private void removeTemplate(DataStoreTO convertLocation, String ovfTemplateOnConvertLocation) {
+        HypervisorGuru vmwareGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.VMware);
+        boolean result = vmwareGuru.removeVMTemplateOutOfBand(convertLocation, ovfTemplateOnConvertLocation);
+        if (!result) {
+            String msg = String.format("Could not remove the template file %s on datastore %s",
+                    ovfTemplateOnConvertLocation, convertLocation.getUrl());
+            logger.warn(msg);
+            return;
+        }
+        logger.debug(String.format("Removed the template file %s on datastore %s",
+                ovfTemplateOnConvertLocation, convertLocation.getUrl()));
+    }
+
     private Map<String, String> createParamsForRemoveClonedInstance(String vcenter, String datacenterName, String username,
                                                                     String password, String sourceVM) {
         Map<String, String> params = new HashMap<>();
@@ -1745,7 +1814,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         return params;
     }
 
-    private HostVO selectInstanceConvertionKVMHostInCluster(Cluster destinationCluster, Long convertInstanceHostId) {
+    private HostVO selectInstanceConversionKVMHostInCluster(Cluster destinationCluster, Long convertInstanceHostId) {
         if (convertInstanceHostId != null) {
             HostVO selectedHost = hostDao.findById(convertInstanceHostId);
             if (selectedHost == null) {
@@ -1762,41 +1831,61 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
             }
             return selectedHost;
         }
-        List<HostVO> hosts = hostDao.listByClusterAndHypervisorType(destinationCluster.getId(), destinationCluster.getHypervisorType());
+        // Auto select host with conversion capability
-        if (CollectionUtils.isEmpty(hosts)) {
+        List<HostVO> hosts = hostDao.listByClusterHypervisorTypeAndHostCapability(destinationCluster.getId(), destinationCluster.getHypervisorType(), Host.HOST_INSTANCE_CONVERSION);
-            String err = String.format("Could not find any running %s host in cluster %s",
+        if (CollectionUtils.isNotEmpty(hosts)) {
+            return hosts.get(new Random().nextInt(hosts.size()));
+        }
+
+        // Try without host capability check
+        hosts = hostDao.listByClusterAndHypervisorType(destinationCluster.getId(), destinationCluster.getHypervisorType());
+        if (CollectionUtils.isNotEmpty(hosts)) {
+            return hosts.get(new Random().nextInt(hosts.size()));
+        }
+
+        String err = String.format("Could not find any suitable %s host in cluster %s to perform the instance conversion",
                 destinationCluster.getHypervisorType(), destinationCluster.getName());
         logger.error(err);
         throw new CloudRuntimeException(err);
     }
-        List<HostVO> filteredHosts = hosts.stream()
+
-                .filter(x -> x.getResourceState() == ResourceState.Enabled)
+    private CheckConvertInstanceAnswer checkConversionSupportOnHost(HostVO convertHost, String sourceVM, boolean checkWindowsGuestConversionSupport) {
-                .collect(Collectors.toList());
+        logger.debug(String.format("Checking the %s conversion support on the host %s (%s)", checkWindowsGuestConversionSupport? "windows guest" : "", convertHost.getId(), convertHost.getName()));
-        if (CollectionUtils.isEmpty(filteredHosts)) {
+        CheckConvertInstanceCommand cmd = new CheckConvertInstanceCommand(checkWindowsGuestConversionSupport);
-            String err = String.format("Could not find a %s host in cluster %s to perform the instance conversion",
+        int timeoutSeconds = 60;
-                    destinationCluster.getHypervisorType(), destinationCluster.getName());
+        cmd.setWait(timeoutSeconds);
+
+        CheckConvertInstanceAnswer checkConvertInstanceAnswer;
+        try {
+            checkConvertInstanceAnswer = (CheckConvertInstanceAnswer) agentManager.send(convertHost.getId(), cmd);
+        } catch (AgentUnavailableException | OperationTimedoutException e) {
+            String err = String.format("Failed to check %s conversion support on the host %s for converting instance %s from VMware to KVM due to: %s",
+                    checkWindowsGuestConversionSupport? "windows guest" : "", convertHost.getName(), sourceVM, e.getMessage());
             logger.error(err);
             throw new CloudRuntimeException(err);
         }
-        return filteredHosts.get(new Random().nextInt(filteredHosts.size()));
+
+        if (!checkConvertInstanceAnswer.getResult()) {
+            String err = String.format("The host %s doesn't support conversion of instance %s from VMware to KVM due to: %s",
+                    convertHost.getName(), sourceVM, checkConvertInstanceAnswer.getDetails());
+            logger.error(err);
+            throw new CloudRuntimeException(err);
         }
 
-    private UnmanagedInstanceTO convertVmwareInstanceToKVM(String vcenter, String datacenterName, String clusterName,
+        return checkConvertInstanceAnswer;
-                                                           String username, String password, String hostName,
+    }
-                                                           UnmanagedInstanceTO clonedInstance, Cluster destinationCluster,
-                                                           Long convertInstanceHostId, Long convertStoragePoolId) {
-        HostVO convertHost = selectInstanceConvertionKVMHostInCluster(destinationCluster, convertInstanceHostId);
-        String vmName = clonedInstance.getName();
-        logger.debug(String.format("The host %s (%s) is selected to execute the conversion of the instance %s" +
-                " from VMware to KVM ", convertHost.getId(), convertHost.getName(), vmName));
 
-        RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(hostName, vmName,
+    private UnmanagedInstanceTO convertVmwareInstanceToKVMWithOVFOnConvertLocation(String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
-                vcenter, datacenterName, clusterName, username, password);
+                                                                                   List<StoragePoolVO> convertStoragePools, DataStoreTO temporaryConvertLocation,
-        DataStoreTO temporaryConvertLocation = selectInstanceConversionTemporaryLocation(destinationCluster, convertStoragePoolId, convertHost);
+                                                                                   String ovfTemplateDirConvertLocation) {
-        List<String> destinationStoragePools = selectInstanceConvertionStoragePools(destinationCluster, clonedInstance.getDisks());
+        logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) using OVF %s on conversion datastore",
+                sourceVM, convertHost.getId(), convertHost.getName(), ovfTemplateDirConvertLocation));
+
+        RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVM);
+        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
         ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
-                Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation);
+                Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, ovfTemplateDirConvertLocation, false, false);
-        int timeoutSeconds = StorageManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
+        int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
         cmd.setWait(timeoutSeconds);
 
         Answer convertAnswer;
@@ -1810,17 +1899,68 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         }
 
         if (!convertAnswer.getResult()) {
-            String err = String.format("The convert process failed for instance %s from Vmware to KVM on host %s: %s",
+            String err = String.format("The convert process failed for instance %s from VMware to KVM on host %s: %s",
-                    vmName, convertHost.getName(), convertAnswer.getDetails());
+                    sourceVM, convertHost.getName(), convertAnswer.getDetails());
             logger.error(err);
             throw new CloudRuntimeException(err);
         }
         return ((ConvertInstanceAnswer) convertAnswer).getConvertedInstance();
     }
 
-    private List<String> selectInstanceConvertionStoragePools(Cluster destinationCluster, List<UnmanagedInstanceTO.Disk> disks) {
+    private UnmanagedInstanceTO convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
+                                                                                             List<StoragePoolVO> convertStoragePools, DataStoreTO temporaryConvertLocation,
+                                                                                             String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
+        logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) after OVF export through ovftool",
+                sourceVM, convertHost.getId(), convertHost.getName()));
+
+        RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVMwareInstance.getName(), vcenterHost, vcenterUsername, vcenterPassword, datacenterName);
+        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
+        ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
+                Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, null, false, true);
+        int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
+        cmd.setWait(timeoutSeconds);
+        int noOfThreads = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles.value();
+        if (noOfThreads == 0) {
+            // Use no. of threads as the disks count
+            noOfThreads = sourceVMwareInstance.getDisks().size();
+        }
+        cmd.setThreadsCountToExportOvf(noOfThreads);
+
+        Answer convertAnswer;
+        try {
+            convertAnswer = agentManager.send(convertHost.getId(), cmd);
+        } catch (AgentUnavailableException | OperationTimedoutException e) {
+            String err = String.format("Could not send the convert instance command to host %s (%s) due to: %s",
+                    convertHost.getId(), convertHost.getName(), e.getMessage());
+            logger.error(err, e);
+            throw new CloudRuntimeException(err);
+        }
+
+        if (!convertAnswer.getResult()) {
+            String err = String.format("The convert process failed for instance %s from VMware to KVM on host %s: %s",
+                    sourceVM, convertHost.getName(), convertAnswer.getDetails());
+            logger.error(err);
+            throw new CloudRuntimeException(err);
+        }
+        return ((ConvertInstanceAnswer) convertAnswer).getConvertedInstance();
+    }
+
+    private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(Cluster destinationCluster) {
+        List<StoragePoolVO> pools = new ArrayList<>();
+        List<StoragePoolVO> clusterPools = primaryDataStoreDao.findClusterWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
+        pools.addAll(clusterPools);
+        List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getDataCenterId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
+        pools.addAll(zonePools);
+        if (pools.isEmpty()) {
+            String msg = String.format("Cannot find suitable storage pools in cluster %s for the conversion", destinationCluster.getName());
+            logger.error(msg);
+            throw new CloudRuntimeException(msg);
+        }
+        return pools;
+    }
+
+    private List<String> selectInstanceConversionStoragePools(List<StoragePoolVO> pools, List<UnmanagedInstanceTO.Disk> disks) {
         List<String> storagePools = new ArrayList<>(disks.size());
-        List<StoragePoolVO> pools = primaryDataStoreDao.listPoolsByCluster(destinationCluster.getId());
         //TODO: Choose pools by capacity
         for (UnmanagedInstanceTO.Disk disk : disks) {
             Long capacity = disk.getCapacity();
@@ -1834,7 +1974,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         throw new CloudRuntimeException(msg);
     }
 
-    protected DataStoreTO selectInstanceConversionTemporaryLocation(Cluster destinationCluster, Long convertStoragePoolId, HostVO convertHost) {
+    protected DataStoreTO selectInstanceConversionTemporaryLocation(Cluster destinationCluster, Long convertStoragePoolId) {
         if (convertStoragePoolId != null) {
             StoragePoolVO selectedStoragePool = primaryDataStoreDao.findById(convertStoragePoolId);
             if (selectedStoragePool == null) {
@@ -1845,11 +1985,10 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
                 logFailureAndThrowException(String.format("Cannot use the storage pool %s for the instance conversion as " +
                         "it is not in the scope of the cluster %s", selectedStoragePool.getName(), destinationCluster.getName()));
             }
-            if (selectedStoragePool.getScope() == ScopeType.HOST &&
+            if (selectedStoragePool.getScope() == ScopeType.HOST) {
-                    storagePoolHostDao.findByPoolHost(selectedStoragePool.getId(), convertHost.getId()) == null) {
+                logFailureAndThrowException(String.format("The storage pool %s is a local storage pool and not supported for temporary conversion location, cluster and zone wide NFS storage pools are supported", selectedStoragePool.getName()));
-                logFailureAndThrowException(String.format("The storage pool %s is not a local storage pool for the host %s", selectedStoragePool.getName(), convertHost.getName()));
             } else if (selectedStoragePool.getPoolType() != Storage.StoragePoolType.NetworkFilesystem) {
-                logFailureAndThrowException(String.format("The storage pool %s is not supported for temporary conversion location, supported pools are NFS storage pools", selectedStoragePool.getName()));
+                logFailureAndThrowException(String.format("The storage pool %s is not supported for temporary conversion location, only NFS storage pools are supported", selectedStoragePool.getName()));
             }
             return dataStoreManager.getPrimaryDataStore(convertStoragePoolId).getTO();
         } else {
@@ -2538,7 +2677,10 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[]{
                 UnmanageVMPreserveNic,
-                RemoteKvmInstanceDisksCopyTimeout
+                RemoteKvmInstanceDisksCopyTimeout,
+                ConvertVmwareInstanceToKvmTimeout,
+                ThreadsOnMSToImportVMwareVMFiles,
+                ThreadsOnKVMHostToImportVMwareVMFiles
         };
     }
 }

server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java

@@ -24,6 +24,8 @@ import com.cloud.dc.dao.DataCenterIpAddressDao;
 import com.cloud.dc.dao.DedicatedResourceDao;
 import com.cloud.dc.dao.HostPodDao;
 import com.cloud.dc.dao.VlanDao;
+import com.cloud.domain.Domain;
+import com.cloud.domain.dao.DomainDao;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.dao.HostDao;
 import com.cloud.network.Network;
@@ -34,63 +36,63 @@ import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.NsxProviderDao;
 import com.cloud.network.dao.PhysicalNetworkDao;
 import com.cloud.network.element.NsxProviderVO;
+import com.cloud.offering.DiskOffering;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.offerings.dao.NetworkOfferingDao;
+import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.dao.VolumeDao;
+import com.cloud.user.Account;
+import com.cloud.user.User;
+import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.net.NetUtils;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
 import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd;
+import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd;
 import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
-import com.cloud.domain.Domain;
-import com.cloud.domain.dao.DomainDao;
-import com.cloud.offering.DiskOffering;
-import com.cloud.storage.DiskOfferingVO;
-import com.cloud.user.Account;
-import com.cloud.user.User;
-import com.cloud.utils.db.EntityManager;
-import com.cloud.utils.db.SearchCriteria;
-import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd;
 import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO;
 import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.vm.UnmanagedVMsManager;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
 
-import java.util.Collections;
-import org.mockito.InjectMocks;
-import org.mockito.Spy;
-
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.lenient;
-import static org.mockito.Mockito.anyString;
 import static org.mockito.Mockito.anyBoolean;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.nullable;
-import static org.mockito.Mockito.anyMap;
-import static org.mockito.Mockito.anyList;
 import static org.mockito.Mockito.anyInt;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.anyList;
+import static org.mockito.Mockito.anyMap;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.nullable;
 import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ConfigurationManagerImplTest {
@@ -109,6 +111,8 @@ public class ConfigurationManagerImplTest {
     @Mock
     Domain domainMock;
     @Mock
+    DataCenterDao zoneDaoMock;
+    @Mock
     DomainDao domainDaoMock;
     @Mock
     EntityManager entityManagerMock;
@@ -457,6 +461,65 @@ public class ConfigurationManagerImplTest {
         Assert.assertNotNull(offering);
     }
 
+    public void testValidateInvalidConfiguration() {
+        Mockito.doReturn(null).when(configDao).findByName(Mockito.anyString());
+        String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Global.toString());
+        Assert.assertEquals("Invalid configuration variable.", msg);
+    }
+
+    @Test
+    public void testValidateInvalidScopeForConfiguration() {
+        ConfigurationVO cfg = mock(ConfigurationVO.class);
+        when(cfg.getScope()).thenReturn(ConfigKey.Scope.Account.toString());
+        Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString());
+        String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Domain.toString());
+        Assert.assertEquals("Invalid scope id provided for the parameter test.config.name", msg);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Failure() {
+        ConfigurationVO cfg = mock(ConfigurationVO.class);
+        when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString());
+        ConfigKey<Integer> configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles;
+        Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString());
+        Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key());
+        configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "11", configKey.scope().toString());
+    }
+
+    @Test
+    public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Success() {
+        ConfigurationVO cfg = mock(ConfigurationVO.class);
+        when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString());
+        ConfigKey<Integer> configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles;
+        Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString());
+        Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key());
+        String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "10", configKey.scope().toString());
+        Assert.assertNull(msg);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Failure() {
+        ConfigurationVO cfg = mock(ConfigurationVO.class);
+        when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString());
+        ConfigKey<Integer> configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout;
+        Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString());
+        Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key());
+        configurationManagerImplSpy.populateConfigValuesForValidationSet();
+        configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "0", configKey.scope().toString());
+    }
+
+    @Test
+    public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Success() {
+        ConfigurationVO cfg = mock(ConfigurationVO.class);
+        when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString());
+        ConfigKey<Integer> configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout;
+        Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString());
+        Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key());
+        configurationManagerImplSpy.populateConfigValuesForValidationSet();
+        String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "9", configKey.scope().toString());
+        Assert.assertNull(msg);
+    }
+
     @Test
     public void validateDomainTestInvalidIdThrowException() {
         Mockito.doReturn(null).when(domainDaoMock).findById(invalidId);