diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 56732dad993..d2d4f165979 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -1127,6 +1127,12 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater logger.error("Error parsing task", e); } } else if (task.getType() == Task.Type.DISCONNECT) { + try { + // an issue has been found if reconnect immediately after disconnecting. please refer to https://github.com/apache/cloudstack/issues/8517 + // wait 5 seconds before reconnecting + Thread.sleep(5000); + } catch (InterruptedException e) { + } reconnect(task.getLink()); return; } else if (task.getType() == Task.Type.OTHER) { diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index b27ba651e4f..8f97edc3935 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -751,7 +751,7 @@ public class AgentProperties{ public static final Property IOTHREADS = new Property<>("iothreads", 1); /** - * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM + * Enable verbose mode for virt-v2v Instance Conversion from VMware to KVM * Data type: Boolean.
* Default value: false */ diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 6e7aa8b21e2..d86eb2a3a7f 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -18,40 +18,39 @@ */ package com.cloud.agent.api.to; +import java.io.Serializable; + import com.cloud.agent.api.LogLevel; import com.cloud.hypervisor.Hypervisor; -import java.io.Serializable; - public class RemoteInstanceTO implements Serializable { private Hypervisor.HypervisorType hypervisorType; - private String hostName; private String instanceName; - // Vmware Remote Instances parameters + // VMware Remote Instances parameters (required for exporting OVA through ovftool) // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes private String vcenterUsername; @LogLevel(LogLevel.Log4jLevel.Off) private String vcenterPassword; private String vcenterHost; private String datacenterName; - private String clusterName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost, - String datacenterName, String clusterName, - String vcenterUsername, String vcenterPassword) { + public RemoteInstanceTO(String instanceName) { + this.hypervisorType = Hypervisor.HypervisorType.VMware; + this.instanceName = instanceName; + } + + public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; - this.hostName = hostName; this.instanceName = instanceName; this.vcenterHost = vcenterHost; - this.datacenterName = datacenterName; - this.clusterName = clusterName; this.vcenterUsername = vcenterUsername; this.vcenterPassword = vcenterPassword; + this.datacenterName = datacenterName; } public Hypervisor.HypervisorType getHypervisorType() { @@ -62,10 +61,6 @@ public class RemoteInstanceTO implements Serializable { return this.instanceName; } - public String getHostName() { - return this.hostName; - } - public String getVcenterUsername() { return vcenterUsername; } @@ -81,8 +76,4 @@ public class RemoteInstanceTO implements Serializable { public String getDatacenterName() { return datacenterName; } - - public String getClusterName() { - return clusterName; - } } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 7563bc3b742..4a3b914364f 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -54,6 +54,7 @@ public interface Host extends StateObject, Identity, Partition, HAResour } public static final String HOST_UEFI_ENABLE = "host.uefi.enable"; public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; /** * @return name of the machine. diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 3c7dbac6442..0c821b4e36c 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -101,21 +102,20 @@ public interface HypervisorGuru extends Adapter { * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware. * * @param vm the stopped vm to migrate - * @param destination the primary storage pool to migrate to + * @param volumeToPool the primary storage pools to migrate to * @return a list of commands to perform for a successful migration */ List finalizeMigrate(VirtualMachine vm, Map volumeToPool); /** - * Will perform a clone of a VM on an external host (if the guru can handle) + * Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle) * @param hostIp VM's source host IP - * @param vmName name of the source VM to clone from + * @param vmName name of the source VM (clone VM name if cloned) * @param params hypervisor specific additional parameters - * @return a reference to the cloned VM + * @return a reference to the hypervisor or cloned VM, and cloned flag */ - UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, - Map params); + Pair getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map params); /** * Removes a VM created as a clone of a VM on an external host @@ -124,6 +124,23 @@ public interface HypervisorGuru extends Adapter { * @param params hypervisor specific additional parameters * @return true if the operation succeeds, false if not */ - boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, - Map params); + boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map params); + + /** + * Create an OVA/OVF template of a VM on an external host (if the guru can handle) + * @param hostIp VM's source host IP + * @param vmName name of the source VM to create template from + * @param params hypervisor specific additional parameters + * @param templateLocation datastore to create the template file + * @return the created template dir/name + */ + String createVMTemplateOutOfBand(String hostIp, String vmName, Map params, DataStoreTO templateLocation, int threadsCountToExportOvf); + + /** + * Removes the template on the location + * @param templateLocation datastore to remove the template file + * @param templateDir the template dir to remove from datastore + * @return true if the operation succeeds, false if not + */ + boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir); } diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index 53ac735cf05..699dcbf6c50 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -317,6 +317,8 @@ public interface NetworkModel { void checkIp6Parameters(String startIPv6, String endIPv6, String ip6Gateway, String ip6Cidr) throws InvalidParameterValueException; + void checkIp6CidrSizeEqualTo64(String ip6Cidr) throws InvalidParameterValueException; + void checkRequestedIpAddresses(long networkId, IpAddresses ips) throws InvalidParameterValueException; String getStartIpv6Address(long id); diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index c47500c7849..cb92739d283 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -17,17 +17,22 @@ package com.cloud.network; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; +import com.cloud.deploy.DeploymentPlanner; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.router.VirtualRouter; import com.cloud.user.Account; import com.cloud.utils.Pair; import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; public interface VirtualNetworkApplianceService { /** @@ -62,6 +67,10 @@ public interface VirtualNetworkApplianceService { VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException; + void startRouterForHA(VirtualMachine vm, Map params, DeploymentPlanner planner) + throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, + OperationTimedoutException; + VirtualRouter destroyRouter(long routerId, Account caller, Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException; VirtualRouter findRouter(long routerId); diff --git a/api/src/main/java/com/cloud/network/guru/NetworkGuru.java b/api/src/main/java/com/cloud/network/guru/NetworkGuru.java index cbadbb18a8f..7b81c75ed84 100644 --- a/api/src/main/java/com/cloud/network/guru/NetworkGuru.java +++ b/api/src/main/java/com/cloud/network/guru/NetworkGuru.java @@ -212,4 +212,7 @@ public interface NetworkGuru extends Adapter { boolean isMyTrafficType(TrafficType type); + default boolean isSlaacV6Only() { + return true; + } } diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index d3c1daa1f5d..183c8dcb2d5 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -62,6 +62,7 @@ public class NicProfile implements InternalIdentity, Serializable { String iPv4Dns1; String iPv4Dns2; String requestedIPv4; + boolean ipv4AllocationRaceCheck; // IPv6 String iPv6Address; @@ -405,6 +406,13 @@ public class NicProfile implements InternalIdentity, Serializable { this.mtu = mtu; } + public boolean getIpv4AllocationRaceCheck() { + return this.ipv4AllocationRaceCheck; + } + + public void setIpv4AllocationRaceCheck(boolean ipv4AllocationRaceCheck) { + this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck; + } // // OTHER METHODS diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index a30c4491f9d..5a9301eef7f 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import com.cloud.dc.DataCenter; +import com.cloud.deploy.DeploymentPlanner; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; @@ -112,6 +113,10 @@ public interface UserVmService { void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException; + void startVirtualMachineForHA(VirtualMachine vm, Map params, + DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException, + ConcurrentOperationException, OperationTimedoutException; + UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException; /** diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index f2ff3da8449..c67ee4eabc2 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -192,6 +192,10 @@ public interface VirtualMachineProfile { Map getParameters(); + void setCpuOvercommitRatio(Float cpuOvercommitRatio); + + void setMemoryOvercommitRatio(Float memoryOvercommitRatio); + Float getCpuOvercommitRatio(); Float getMemoryOvercommitRatio(); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index bad462e5a00..a2bcd103b76 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -191,6 +191,7 @@ public class ApiConstants { public static final String FORCED = "forced"; public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage"; public static final String FORCE_DELETE_HOST = "forcedeletehost"; + public static final String FORCE_MS_TO_IMPORT_VM_FILES = "forcemstoimportvmfiles"; public static final String FORMAT = "format"; public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; @@ -239,6 +240,7 @@ public class ApiConstants { public static final String NEXT_ACL_RULE_ID = "nextaclruleid"; public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; public static final String IMAGE_PATH = "imagepath"; + public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported"; public static final String INTERNAL_DNS1 = "internaldns1"; public static final String INTERNAL_DNS2 = "internaldns2"; public static final String INTERNET_PROTOCOL = "internetprotocol"; @@ -1135,6 +1137,8 @@ public class ApiConstants { public static final String WEBHOOK_ID = "webhookid"; public static final String WEBHOOK_NAME = "webhookname"; + public static final String NFS_MOUNT_OPTIONS = "nfsmountopts"; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 1a34b7ea6cc..6f148ff0ee4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VmwareDatacenterResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.vm.VmImportService; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -116,40 +117,44 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "Temp Path on external host for disk image copy" ) private String tmpPath; - // Import from Vmware to KVM migration parameters + // Import from VMware to KVM migration parameters @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, type = CommandType.UUID, entityType = VmwareDatacenterResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter") + description = "(only for importing VMs from VMware to KVM) UUID of a linked existing vCenter") private Long existingVcenterId; @Parameter(name = ApiConstants.HOST_IP, type = BaseCmd.CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.") + description = "(only for importing VMs from VMware to KVM) VMware ESXi host IP/Name.") private String hostip; @Parameter(name = ApiConstants.VCENTER, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + description = "(only for importing VMs from VMware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vcenter; @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.") + description = "(only for importing VMs from VMware to KVM) Name of VMware datacenter.") private String datacenterName; @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.") + description = "(only for importing VMs from VMware to KVM) Name of VMware cluster.") private String clusterName; @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") private Long convertInstanceHostId; @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") private Long convertStoragePoolId; + @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN, + description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") + private Boolean forceMsToImportVmFiles; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -198,6 +203,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return convertStoragePoolId; } + public Boolean getForceMsToImportVmFiles() { + return BooleanUtils.toBooleanDefaultIfNull(forceMsToImportVmFiles, false); + } + public String getHypervisor() { return hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java index 81a52ce2dfe..c4424b1d937 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java @@ -72,7 +72,7 @@ public class AssignToLoadBalancerRuleCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID_IP, type = CommandType.MAP, - description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].ip=10.1.1.75", + description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].vmip=10.1.1.75", since = "4.4") private Map vmIdIpMap; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 3a88b819572..8d98b6573f4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; import com.cloud.host.Host; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -285,6 +286,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "true if the host supports encryption", since = "4.18") private Boolean encryptionSupported; + @SerializedName(ApiConstants.INSTANCE_CONVERSION_SUPPORTED) + @Param(description = "true if the host supports instance conversion (using virt-v2v)", since = "4.19.1") + private Boolean instanceConversionSupported; + @Override public String getObjectId() { return this.getId(); @@ -550,7 +555,7 @@ public class HostResponse extends BaseResponseWithAnnotations { this.username = username; } - public void setDetails(Map details) { + public void setDetails(Map details, Hypervisor.HypervisorType hypervisorType) { if (details == null) { return; @@ -571,6 +576,15 @@ public class HostResponse extends BaseResponseWithAnnotations { this.setEncryptionSupported(new Boolean(false)); // default } + if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + if (detailsCopy.containsKey(Host.HOST_INSTANCE_CONVERSION)) { + this.setInstanceConversionSupported(Boolean.parseBoolean((String) detailsCopy.get(Host.HOST_INSTANCE_CONVERSION))); + detailsCopy.remove(Host.HOST_INSTANCE_CONVERSION); + } else { + this.setInstanceConversionSupported(new Boolean(false)); // default + } + } + this.details = detailsCopy; } @@ -761,6 +775,10 @@ public class HostResponse extends BaseResponseWithAnnotations { this.encryptionSupported = encryptionSupported; } + public void setInstanceConversionSupported(Boolean instanceConversionSupported) { + this.instanceConversionSupported = instanceConversionSupported; + } + public Boolean getIsTagARule() { return isTagARule; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 183290ec9eb..f514c8167ac 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.NFS_MOUNT_OPTIONS) + @Param(description = "the nfs mount options for the storage pool", since = "4.19.1") + private String nfsMountOpts; + @SerializedName(ApiConstants.IS_TAG_A_RULE) @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) private Boolean isTagARule; @@ -347,4 +351,12 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { public void setProvider(String provider) { this.provider = provider; } + + public String getNfsMountOpts() { + return nfsMountOpts; + } + + public void setNfsMountOpts(String nfsMountOpts) { + this.nfsMountOpts = nfsMountOpts; + } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 23e0e371714..5697a040b81 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.vm; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.util.List; public class UnmanagedInstanceTO { @@ -317,6 +319,16 @@ public class UnmanagedInstanceTO { public int getDatastorePort() { return datastorePort; } + + @Override + public String toString() { + return "Disk {" + + "diskId='" + diskId + '\'' + + ", capacity=" + toHumanReadableSize(capacity) + + ", controller='" + controller + '\'' + + ", controllerUnit=" + controllerUnit + + "}"; + } } public static class Nic { @@ -409,5 +421,14 @@ public class UnmanagedInstanceTO { public void setPciSlot(String pciSlot) { this.pciSlot = pciSlot; } + + @Override + public String toString() { + return "Nic{" + + "nicId='" + nicId + '\'' + + ", adapterType='" + adapterType + '\'' + + ", macAddress='" + macAddress + '\'' + + "}"; + } } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java index e3a484b35b3..b6233b9c270 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java @@ -39,6 +39,37 @@ public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService, ConfigKey.Scope.Global, null); + ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, + "convert.vmware.instance.to.kvm.timeout", + "Advanced", + "3", + "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", + true, + ConfigKey.Scope.Global, + null); + + ConfigKey ThreadsOnMSToImportVMwareVMFiles = new ConfigKey<>(Integer.class, + "threads.on.ms.to.import.vmware.vm.files", + "Advanced", + "0", + "Threads to use on the management server when importing VM files from VMWare." + + " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." + + " Max number is 10, Default is 0.", + true, + ConfigKey.Scope.Global, + null); + + ConfigKey ThreadsOnKVMHostToImportVMwareVMFiles = new ConfigKey<>(Integer.class, + "threads.on.kvm.host.to.import.vmware.vm.files", + "Advanced", + "0", + "Threads to use on the KVM host (by the ovftool, if the version is 4.4.0+) when importing VM files from VMWare." + + " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." + + " Max number is 10, Default is 0.", + true, + ConfigKey.Scope.Global, + null); + static boolean isSupported(Hypervisor.HypervisorType hypervisorType) { return hypervisorType == VMware || hypervisorType == KVM; } diff --git a/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java index 523b3de9e3c..04e3ad7df60 100644 --- a/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java @@ -23,6 +23,8 @@ import org.junit.Test; import java.util.HashMap; import java.util.Map; +import com.cloud.hypervisor.Hypervisor; + public final class HostResponseTest extends TestCase { private static final String VALID_KEY = "validkey"; @@ -32,7 +34,7 @@ public final class HostResponseTest extends TestCase { public void testSetDetailsNull() { final HostResponse hostResponse = new HostResponse(); - hostResponse.setDetails(null); + hostResponse.setDetails(null, null); assertEquals(null, hostResponse.getDetails()); @@ -51,7 +53,7 @@ public final class HostResponseTest extends TestCase { final Map expectedDetails = new HashedMap(); expectedDetails.put(VALID_KEY, VALID_VALUE); - hostResponse.setDetails(details); + hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM); final Map actualDetails = hostResponse.getDetails(); assertTrue(details != actualDetails); @@ -70,7 +72,7 @@ public final class HostResponseTest extends TestCase { final Map expectedDetails = new HashedMap(); expectedDetails.put(VALID_KEY, VALID_VALUE); - hostResponse.setDetails(details); + hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM); final Map actualDetails = hostResponse.getDetails(); assertTrue(details != actualDetails); diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java new file mode 100644 index 00000000000..a02ac92927a --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class CheckConvertInstanceAnswer extends Answer { + + private boolean ovfExportSupported = false; + + public CheckConvertInstanceAnswer() { + super(); + } + + public CheckConvertInstanceAnswer(Command command, boolean success) { + super(command, success, ""); + } + + public CheckConvertInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public CheckConvertInstanceAnswer(Command command, boolean success, boolean ovfExportSupported, String details) { + super(command, success, details); + this.ovfExportSupported = ovfExportSupported; + } + + public boolean isOvfExportSupported() { + return ovfExportSupported; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java new file mode 100644 index 00000000000..fc066e5c589 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class CheckConvertInstanceCommand extends Command { + boolean checkWindowsGuestConversionSupport = false; + + public CheckConvertInstanceCommand() { + } + + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public boolean getCheckWindowsGuestConversionSupport() { + return checkWindowsGuestConversionSupport; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 63234b04480..b8250903f85 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -28,16 +28,24 @@ public class ConvertInstanceCommand extends Command { private Hypervisor.HypervisorType destinationHypervisorType; private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; + private String templateDirOnConversionLocation; + private boolean checkConversionSupport; + private boolean exportOvfToConversionLocation; + private int threadsCountToExportOvf = 0; public ConvertInstanceCommand() { } public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation) { + List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; + this.templateDirOnConversionLocation = templateDirOnConversionLocation; + this.checkConversionSupport = checkConversionSupport; + this.exportOvfToConversionLocation = exportOvfToConversionLocation; } public RemoteInstanceTO getSourceInstance() { @@ -56,6 +64,26 @@ public class ConvertInstanceCommand extends Command { return conversionTemporaryLocation; } + public String getTemplateDirOnConversionLocation() { + return templateDirOnConversionLocation; + } + + public boolean getCheckConversionSupport() { + return checkConversionSupport; + } + + public boolean getExportOvfToConversionLocation() { + return exportOvfToConversionLocation; + } + + public int getThreadsCountToExportOvf() { + return threadsCountToExportOvf; + } + + public void setThreadsCountToExportOvf(int threadsCountToExportOvf) { + this.threadsCountToExportOvf = threadsCountToExportOvf; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java index ad05fe1d615..06940266b53 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java @@ -46,6 +46,10 @@ public class ModifyStoragePoolCommand extends Command { this.details = details; } + public ModifyStoragePoolCommand(boolean add, StoragePool pool, Map details) { + this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes()), details); + } + public ModifyStoragePoolCommand(boolean add, StoragePool pool) { this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes())); } diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 9308be5fb32..91197de6a84 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -126,6 +126,8 @@ public interface ResourceManager extends ResourceService, Configurable { public List listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType type, long dcId); + public List listAllUpHostsInOneZoneByHypervisor(HypervisorType type, long dcId); + public List listAllUpAndEnabledHostsInOneZone(long dcId); public List listAllHostsInOneZoneByType(Host.Type type, long dcId); diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 6653bb2bad3..779c08e1505 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -18,6 +18,7 @@ package com.cloud.storage; import java.math.BigDecimal; import java.util.List; +import java.util.Map; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -96,14 +97,6 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); - ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, - "convert.vmware.instance.to.kvm.timeout", - "Storage", - "8", - "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", - true, - ConfigKey.Scope.Global, - null); ConfigKey KvmAutoConvergence = new ConfigKey<>(Boolean.class, "kvm.auto.convergence", "Storage", @@ -348,6 +341,10 @@ public interface StorageManager extends StorageService { boolean registerHostListener(String providerUuid, HypervisorHostListener listener); + Pair, Boolean> getStoragePoolNFSMountOpts(StoragePool pool, Map details); + + String getStoragePoolMountFailureReason(String error); + boolean connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; diff --git a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java index 2d51c3c0870..a1c54b90328 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java @@ -264,11 +264,13 @@ public class VirtualMachineProfileImpl implements VirtualMachineProfile { _offering = offering; } + @Override public void setCpuOvercommitRatio(Float cpuOvercommitRatio) { this.cpuOvercommitRatio = cpuOvercommitRatio; } + @Override public void setMemoryOvercommitRatio(Float memoryOvercommitRatio) { this.memoryOvercommitRatio = memoryOvercommitRatio; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 22c0b3fd71a..173fd9fc704 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -45,6 +45,8 @@ import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.CleanupNetworkRulesCmd; import com.cloud.agent.api.Command; +import com.cloud.agent.api.CreateStoragePoolCommand; +import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.MaintainCommand; import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.ModifySshKeysCommand; @@ -122,8 +124,9 @@ public abstract class AgentAttache { StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(), ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(), CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(), - ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), - CleanupPersistentNetworkResourceCommand.class.toString()}; + ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), + CreateStoragePoolCommand.class.toString(), DeleteStoragePoolCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), + SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), CleanupPersistentNetworkResourceCommand.class.toString()}; protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() }; static { Arrays.sort(s_commandsAllowedInMaintenanceMode); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e2e11965d3e..d21e8b0fc7b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1230,21 +1230,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long destHostId = dest.getHost().getId(); vm.setPodIdToDeployIn(dest.getPod().getId()); - final Long cluster_id = dest.getCluster().getId(); - final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); - final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + final Long clusterId = dest.getCluster().getId(); + updateOverCommitRatioForVmProfile(vmProfile, clusterId); - if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null && - (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } - - vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); - vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue())); StartAnswer startAnswer = null; try { @@ -1259,7 +1247,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac resetVmNicsDeviceId(vm.getId()); _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { - checkAndAttemptMigrateVmAcrossCluster(vm, cluster_id, dest.getStorageForDisks()); + checkAndAttemptMigrateVmAcrossCluster(vm, clusterId, dest.getStorageForDisks()); volumeMgr.prepare(vmProfile, dest); } @@ -1504,6 +1492,27 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac networkToNetworkNameMap.put(networkVO.getId(), networkName); } + private void updateOverCommitRatioForVmProfile(VirtualMachineProfile vmProfile, long clusterId) { + final ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO); + final ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + final float parsedClusterCpuDetailCpu = Float.parseFloat(clusterDetailCpu.getValue()); + final float parsedClusterDetailRam = Float.parseFloat(clusterDetailRam.getValue()); + UserVmDetailVO vmDetailCpu = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); + UserVmDetailVO vmDetailRam = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + + if ((vmDetailCpu == null && parsedClusterCpuDetailCpu > 1f) || + (vmDetailCpu != null && Float.parseFloat(vmDetailCpu.getValue()) != parsedClusterCpuDetailCpu)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterDetailCpu.getValue(), true); + } + if ((vmDetailRam == null && parsedClusterDetailRam > 1f) || + (vmDetailRam != null && Float.parseFloat(vmDetailRam.getValue()) != parsedClusterDetailRam)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterDetailRam.getValue(), true); + } + + vmProfile.setCpuOvercommitRatio(Float.parseFloat(clusterDetailCpu.getValue())); + vmProfile.setMemoryOvercommitRatio(Float.parseFloat(clusterDetailRam.getValue())); + } + /** * Setting pod id to null can result in migration of Volumes across pods. This is not desirable for VMs which * have a volume in Ready state (happens when a VM is shutdown and started again). @@ -2001,20 +2010,24 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void updatePersistenceMap(Map vlanToPersistenceMap, NetworkVO networkVO) { + if (networkVO == null) { + return; + } NetworkOfferingVO offeringVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId()); - if (offeringVO != null) { - Pair data = getVMNetworkDetails(networkVO, offeringVO.isPersistent()); - Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null; - if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) { - vlanToPersistenceMap.put(data.first(), data.second()); - } + if (offeringVO == null) { + return; + } + Pair data = getVMNetworkDetails(networkVO, offeringVO.isPersistent()); + Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null; + if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) { + vlanToPersistenceMap.put(data.first(), data.second()); } } private Map getVlanToPersistenceMapForVM(long vmId) { List userVmJoinVOs = userVmJoinDao.searchByIds(vmId); Map vlanToPersistenceMap = new HashMap<>(); - if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { + if (CollectionUtils.isNotEmpty(userVmJoinVOs)) { for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) { NetworkVO networkVO = _networkDao.findById(userVmJoinVO.getNetworkId()); updatePersistenceMap(vlanToPersistenceMap, networkVO); @@ -2728,6 +2741,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _networkMgr.prepareNicForMigration(profile, dest); volumeMgr.prepareForMigration(profile, dest); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); + updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId()); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); @@ -4777,6 +4791,18 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + private ApiCommandResourceType getApiCommandResourceTypeForVm(VirtualMachine vm) { + switch (vm.getType()) { + case DomainRouter: + return ApiCommandResourceType.DomainRouter; + case ConsoleProxy: + return ApiCommandResourceType.ConsoleProxy; + case SecondaryStorageVm: + return ApiCommandResourceType.SystemVm; + } + return ApiCommandResourceType.VirtualMachine; + } + private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { Host host = _hostDao.findById(vm.getHostId()); Host poweredHost = _hostDao.findById(vm.getPowerHostId()); @@ -4824,7 +4850,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac + " -> Running) from out-of-context transition. VM network environment may need to be reset"); ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, vm.getDomainId(), - EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), ApiCommandResourceType.VirtualMachine.toString()); + EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), getApiCommandResourceTypeForVm(vm).toString()); logger.info("VM {} is sync-ed to at Running state according to power-on report from hypervisor.", vm.getInstanceName()); break; @@ -4857,7 +4883,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac case Running: case Stopped: ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,vm.getDomainId(), - EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), ApiCommandResourceType.VirtualMachine.toString()); + EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), getApiCommandResourceTypeForVm(vm).toString()); case Migrating: logger.info("VM {} is at {} and we received a {} report while there is no pending jobs on it" , vm.getInstanceName(), vm.getState(), vm.getPowerState()); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index e589f688cdb..ea34f62ecd5 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -765,6 +765,14 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra continue; } + // Ensure cidr size is equal to 64 for + // - networks other than shared networks + // - shared networks with SLAAC V6 only + if (predefined != null && StringUtils.isNotBlank(predefined.getIp6Cidr()) && + (!GuestType.Shared.equals(offering.getGuestType()) || guru.isSlaacV6Only())) { + _networkModel.checkIp6CidrSizeEqualTo64(predefined.getIp6Cidr()); + } + if (network.getId() != -1) { if (network instanceof NetworkVO) { networks.add((NetworkVO) network); @@ -1033,48 +1041,84 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } + private NicVO persistNicAfterRaceCheck(final NicVO nic, final Long networkId, final NicProfile profile, int deviceId) { + return Transaction.execute(new TransactionCallback() { + @Override + public NicVO doInTransaction(TransactionStatus status) { + NicVO vo = _nicDao.findByIp4AddressAndNetworkId(profile.getIPv4Address(), networkId); + if (vo == null) { + applyProfileToNic(nic, profile, deviceId); + vo = _nicDao.persist(nic); + return vo; + } else { + return null; + } + } + }); + } + + private NicVO checkForRaceAndAllocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException { + final NetworkVO ntwkVO = _networksDao.findById(network.getId()); + logger.debug("Allocating nic for vm {} in network {} with requested profile {}", vm.getVirtualMachine(), network, requested); + final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName()); + + NicVO vo = null; + boolean retryIpAllocation; + do { + retryIpAllocation = false; + final NicProfile profile = guru.allocate(network, requested, vm); + if (profile == null) { + return null; + } + + if (isDefaultNic != null) { + profile.setDefaultNic(isDefaultNic); + } + + if (requested != null && requested.getMode() == null) { + profile.setMode(requested.getMode()); + } else { + profile.setMode(network.getMode()); + } + + vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType()); + + DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId()); + if (dcVo.getNetworkType() == NetworkType.Basic) { + configureNicProfileBasedOnRequestedIp(requested, profile, network); + } + + if (profile.getIpv4AllocationRaceCheck()) { + vo = persistNicAfterRaceCheck(vo, network.getId(), profile, deviceId); + } else { + applyProfileToNic(vo, profile, deviceId); + vo = _nicDao.persist(vo); + } + + if (vo == null) { + if (requested.getRequestedIPv4() != null) { + throw new InsufficientVirtualNetworkCapacityException("Unable to acquire requested Guest IP address " + requested.getRequestedIPv4() + " for network " + network, DataCenter.class, dcVo.getId()); + } else { + requested.setIPv4Address(null); + } + retryIpAllocation = true; + } + } while (retryIpAllocation); + + return vo; + } + @DB @Override public Pair allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { - final NetworkVO ntwkVO = _networksDao.findById(network.getId()); - logger.debug("Allocating nic for vm {} in network {} with requested profile {}", vm.getVirtualMachine(), network, requested); - final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName()); - if (requested != null && requested.getMode() == null) { requested.setMode(network.getMode()); } - final NicProfile profile = guru.allocate(network, requested, vm); - if (profile == null) { - return null; - } - if (isNicAllocatedForNsxPublicNetworkOnVR(network, profile, vm)) { - String guruName = "NsxPublicNetworkGuru"; - NetworkGuru nsxGuru = AdapterBase.getAdapterByName(networkGurus, guruName); - nsxGuru.allocate(network, profile, vm); - } - - if (isDefaultNic != null) { - profile.setDefaultNic(isDefaultNic); - } - - if (requested != null && requested.getMode() == null) { - profile.setMode(requested.getMode()); - } else { - profile.setMode(network.getMode()); - } - - NicVO vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType()); - - DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId()); - if (dcVo.getNetworkType() == NetworkType.Basic) { - configureNicProfileBasedOnRequestedIp(requested, profile, network); - } - - deviceId = applyProfileToNic(vo, profile, deviceId); - vo = _nicDao.persist(vo); + NicVO vo = checkForRaceAndAllocateNic(requested, network, isDefaultNic, deviceId, vm); final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId()); final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network), @@ -2723,8 +2767,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - if (ipv6 && NetUtils.getIp6CidrSize(ip6Cidr) != 64) { - throw new InvalidParameterValueException("IPv6 subnet should be exactly 64-bits in size"); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } //TODO(VXLAN): Support VNI specified @@ -4605,10 +4649,16 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NicVO vo = Transaction.execute(new TransactionCallback() { @Override public NicVO doInTransaction(TransactionStatus status) { - NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddress); - String macAddressToPersist = macAddress; + if (StringUtils.isBlank(macAddress)) { + throw new CloudRuntimeException("Mac address not specified"); + } + String macAddressToPersist = macAddress.trim(); + if (!NetUtils.isValidMac(macAddressToPersist)) { + throw new CloudRuntimeException("Invalid mac address: " + macAddressToPersist); + } + NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddressToPersist); if (existingNic != null) { - macAddressToPersist = generateNewMacAddressIfForced(network, macAddress, forced); + macAddressToPersist = generateNewMacAddressIfForced(network, macAddressToPersist, forced); } NicVO vo = new NicVO(network.getGuruName(), vm.getId(), network.getId(), vm.getType()); vo.setMacAddress(macAddressToPersist); @@ -4653,7 +4703,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vm.getHypervisorType(), network)); - return new Pair(vmNic, Integer.valueOf(deviceId)); + return new Pair<>(vmNic, Integer.valueOf(deviceId)); } protected String getSelectedIpForNicImport(Network network, DataCenter dataCenter, Network.IpAddresses ipAddresses) { @@ -4697,7 +4747,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra private String generateNewMacAddressIfForced(Network network, String macAddress, boolean forced) { if (!forced) { - throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + + throw new CloudRuntimeException("NIC with MAC address " + macAddress + " exists on network with ID " + network.getUuid() + " and forced flag is disabled"); } try { diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java index a4700f6cdc0..58746a9a6cf 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.engine.orchestration; import static org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService.NetworkLockTimeout; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import com.cloud.dc.DataCenter; +import com.cloud.exception.InsufficientVirtualNetworkCapacityException; import com.cloud.network.IpAddressManager; import com.cloud.utils.Pair; import org.junit.Assert; @@ -39,6 +41,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; import org.mockito.Mockito; import com.cloud.api.query.dao.DomainRouterJoinDao; @@ -71,6 +74,8 @@ import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.vm.DomainRouterVO; @@ -890,4 +895,118 @@ public class NetworkOrchestratorTest extends TestCase { verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value()); verify(testOrchestrator._networksDao, times(1)).releaseFromLockTable(networkId); } + + @Test(expected = InsufficientVirtualNetworkCapacityException.class) + public void testImportNicAcquireGuestIPFailed() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + VirtualMachine vm = mock(VirtualMachine.class); + Network network = Mockito.mock(Network.class); + Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated); + Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId); + long dataCenterId = 1L; + Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + String ipAddress = "10.1.10.10"; + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null); + Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 0; + testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + } + + @Test(expected = InsufficientVirtualNetworkCapacityException.class) + public void testImportNicAutoAcquireGuestIPFailed() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + VirtualMachine vm = mock(VirtualMachine.class); + Network network = Mockito.mock(Network.class); + Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated); + Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId); + long dataCenterId = 1L; + Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + String ipAddress = "auto"; + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null); + Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 0; + testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + } + + @Test + public void testImportNicNoIP4Address() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + Long vmId = 1L; + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + VirtualMachine vm = mock(VirtualMachine.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType); + Long networkId = 1L; + Network network = Mockito.mock(Network.class); + Mockito.when(network.getId()).thenReturn(networkId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + Mockito.when(ipAddresses.getIp4Address()).thenReturn(null); + URI broadcastUri = URI.create("vlan://123"); + NicVO nic = mock(NicVO.class); + Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 1; + Integer networkRate = 200; + Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); + Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); + Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag"); + try (MockedStatic transactionMocked = Mockito.mockStatic(Transaction.class)) { + transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic); + Pair nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId); + verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network); + verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network); + assertEquals(deviceId, nicProfileIntegerPair.second().intValue()); + NicProfile nicProfile = nicProfileIntegerPair.first(); + assertEquals(broadcastUri, nicProfile.getBroadCastUri()); + assertEquals(networkRate, nicProfile.getNetworkRate()); + assertFalse(nicProfile.isSecurityGroupEnabled()); + assertEquals("testtag", nicProfile.getName()); + } + } + + @Test + public void testImportNicWithIP4Address() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + Long vmId = 1L; + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + VirtualMachine vm = mock(VirtualMachine.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType); + Long networkId = 1L; + Network network = Mockito.mock(Network.class); + Mockito.when(network.getId()).thenReturn(networkId); + String ipAddress = "10.1.10.10"; + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + URI broadcastUri = URI.create("vlan://123"); + NicVO nic = mock(NicVO.class); + Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 1; + Integer networkRate = 200; + Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); + Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); + Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag"); + try (MockedStatic transactionMocked = Mockito.mockStatic(Transaction.class)) { + transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic); + Pair nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + verify(testOrchestrator, times(1)).getSelectedIpForNicImport(network, dataCenter, ipAddresses); + verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId); + verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network); + verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network); + assertEquals(deviceId, nicProfileIntegerPair.second().intValue()); + NicProfile nicProfile = nicProfileIntegerPair.first(); + assertEquals(broadcastUri, nicProfile.getBroadCastUri()); + assertEquals(networkRate, nicProfile.getNetworkRate()); + assertFalse(nicProfile.isSecurityGroupEnabled()); + assertEquals("testtag", nicProfile.getName()); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca180e2323f..08380ed8b40 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -141,6 +141,8 @@ public interface HostDao extends GenericDao, StateDao listByHostCapability(Host.Type type, Long clusterId, Long podId, long dcId, String hostCapabilty); + List listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty); + List listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType); HostVO findByName(String name); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 5faa877b458..170c6a45fc3 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -341,6 +341,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao ClusterHypervisorSearch.and("hypervisor", ClusterHypervisorSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.and("type", ClusterHypervisorSearch.entity().getType(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.and("status", ClusterHypervisorSearch.entity().getStatus(), SearchCriteria.Op.EQ); + ClusterHypervisorSearch.and("resourceState", ClusterHypervisorSearch.entity().getResourceState(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.done(); UnmanagedDirectConnectSearch = createSearchBuilder(); @@ -1506,12 +1507,42 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty) { + SearchBuilder hostCapabilitySearch = _detailsDao.createSearchBuilder(); + DetailVO tagEntity = hostCapabilitySearch.entity(); + hostCapabilitySearch.and("capability", tagEntity.getName(), SearchCriteria.Op.EQ); + hostCapabilitySearch.and("value", tagEntity.getValue(), SearchCriteria.Op.EQ); + + SearchBuilder hostSearch = createSearchBuilder(); + HostVO entity = hostSearch.entity(); + hostSearch.and("clusterId", entity.getClusterId(), SearchCriteria.Op.EQ); + hostSearch.and("hypervisor", entity.getHypervisorType(), SearchCriteria.Op.EQ); + hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ); + hostSearch.and("status", entity.getStatus(), SearchCriteria.Op.EQ); + hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ); + hostSearch.join("hostCapabilitySearch", hostCapabilitySearch, entity.getId(), tagEntity.getHostId(), JoinBuilder.JoinType.INNER); + + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostCapabilitySearch", "value", Boolean.toString(true)); + sc.setJoinParameters("hostCapabilitySearch", "capability", hostCapabilty); + + sc.setParameters("clusterId", clusterId); + sc.setParameters("hypervisor", hypervisorType); + sc.setParameters("type", Type.Routing); + sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); + return listBy(sc); + } + + @Override public List listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType) { SearchCriteria sc = ClusterHypervisorSearch.create(); sc.setParameters("clusterId", clusterId); sc.setParameters("hypervisor", hypervisorType); sc.setParameters("type", Type.Routing); sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); return listBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java index 43bb5b3d4d5..02a0355d92d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java @@ -18,9 +18,12 @@ */ package com.cloud.storage.dao; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; import com.cloud.utils.db.GenericDao; public interface SnapshotDetailsDao extends GenericDao, ResourceDetailsDao { + public List findDetailsByZoneAndKey(long dcId, String key); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java index e4ae22cd021..584a2481726 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java @@ -18,11 +18,44 @@ */ package com.cloud.storage.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; + public class SnapshotDetailsDaoImpl extends ResourceDetailsDaoBase implements SnapshotDetailsDao { + private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE ss.data_center_id = ? AND s.name = ?"; + @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new SnapshotDetailsVO(resourceId, key, value, display)); } + + public List findDetailsByZoneAndKey(long dcId, String key) { + StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + List snapshotDetailsOnZone = new ArrayList(); + try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) { + if (pstmt != null) { + pstmt.setLong(1, dcId); + pstmt.setString(2, key); + try (ResultSet rs = pstmt.executeQuery();) { + while (rs.next()) { + snapshotDetailsOnZone.add(toEntityBean(rs, false)); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } + return snapshotDetailsOnZone; + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index db1563f3ff1..e6ffca06f9e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -160,4 +160,6 @@ public interface VolumeDao extends GenericDao, StateDao listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(long accountId, List diskOfferingIds, List vmIds); List searchRemovedByVms(List vmIds, Long batchSize); + + VolumeVO findOneByIScsiName(String iScsiName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 551ff7d2d78..16004ac13b9 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -395,6 +395,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol AllFieldsSearch.and("updatedCount", AllFieldsSearch.entity().getUpdatedCount(), Op.EQ); AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), Op.EQ); AllFieldsSearch.and("passphraseId", AllFieldsSearch.entity().getPassphraseId(), Op.EQ); + AllFieldsSearch.and("iScsiName", AllFieldsSearch.entity().get_iScsiName(), Op.EQ); AllFieldsSearch.done(); RootDiskStateSearch = createSearchBuilder(); @@ -910,4 +911,10 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol Filter filter = new Filter(VolumeVO.class, "id", true, 0L, batchSize); return searchIncludingRemoved(sc, filter, null, false); } + + public VolumeVO findOneByIScsiName(String iScsiName) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("iScsiName", iScsiName); + return findOneIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java index ff6af56c9c3..b4a596dfc8d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java @@ -75,8 +75,10 @@ public interface VolumeStatsDao extends GenericDao { /** * Removes (expunges) all Volume stats with {@code timestamp} less than * a given Date. - * @param limit the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query. + * If 0 or negative, no limit is used. */ - void removeAllByTimestampLessThan(Date limit); + void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java index 5d0d3c8921c..d1149e47408 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java @@ -21,6 +21,8 @@ import java.util.List; import javax.annotation.PostConstruct; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -33,6 +35,8 @@ import com.cloud.storage.VolumeStatsVO; @Component public class VolumeStatsDaoImpl extends GenericDaoBase implements VolumeStatsDao { + protected Logger logger = LogManager.getLogger(getClass()); + protected SearchBuilder volumeIdSearch; protected SearchBuilder volumeIdTimestampGreaterThanEqualSearch; protected SearchBuilder volumeIdTimestampLessThanEqualSearch; @@ -116,9 +120,21 @@ public class VolumeStatsDaoImpl extends GenericDaoBase impl } @Override - public void removeAllByTimestampLessThan(Date limit) { + public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) { SearchCriteria sc = timestampSearch.create(); - sc.setParameters(TIMESTAMP, limit); - expunge(sc); + sc.setParameters(TIMESTAMP, limitDate); + + logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate)); + + long totalRemoved = 0; + long removed; + + do { + removed = expunge(sc, limitPerQuery); + totalRemoved += removed; + logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate)); + } while (limitPerQuery > 0 && removed >= limitPerQuery); + + logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate)); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index d42e863cbed..f0c235e842c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -127,6 +127,10 @@ public interface PrimaryDataStoreDao extends GenericDao { List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType, String keyword); + List findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType); + + List findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType); + List findLocalStoragePoolsByHostAndTags(long hostId, String[] tags); List listLocalStoragePoolByPath(long datacenterId, String path); @@ -141,6 +145,8 @@ public interface PrimaryDataStoreDao extends GenericDao { List findPoolsByStorageType(Storage.StoragePoolType storageType); + StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath); + List listStoragePoolsWithActiveVolumesByOfferingId(long offeringid); Pair, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId, diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index e4dd66a86b8..1658fe0a537 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Storage; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import org.apache.commons.collections.CollectionUtils; @@ -35,7 +36,6 @@ import org.apache.commons.collections.CollectionUtils; import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StoragePoolTagVO; @@ -622,6 +622,28 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase return sc.list(); } + @Override + public List findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getDataCenterId(), Op.EQ, dataCenterId); + sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up); + sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE); + sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType); + sc.and(sc.entity().getPoolType(), Op.EQ, poolType); + return sc.list(); + } + + @Override + public List findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getClusterId(), Op.EQ, clusterId); + sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up); + sc.and(sc.entity().getScope(), Op.EQ, ScopeType.CLUSTER); + sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType); + sc.and(sc.entity().getPoolType(), Op.EQ, poolType); + return sc.list(); + } + @Override public void deletePoolTags(long poolId) { _tagsDao.deleteTags(poolId); @@ -660,6 +682,16 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase return listBy(sc); } + @Override + public StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath) { + SearchCriteria sc = AllFieldSearch.create(); + sc.setParameters("datacenterId", zoneId); + if (datastorePath != null) { + sc.addAnd("path", Op.LIKE, "%/" + datastorePath); + } + return findOneBy(sc); + } + @Override public List listStoragePoolsWithActiveVolumesByOfferingId(long offeringId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index ba783e81586..5109118fb54 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -100,6 +100,9 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Override public TemplateInfo getTemplate(long templateId, DataStore store) { VMTemplateVO templ = imageDataDao.findById(templateId); + if (templ == null) { + return null; + } if (store == null && !templ.isDirectDownload()) { TemplateObject tmpl = TemplateObject.getTemplate(templ, null, null); return tmpl; diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index fdb4fe6753a..37ddc3573c1 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -82,6 +82,10 @@ public class TemplateObject implements TemplateInfo { } protected void configure(VMTemplateVO template, DataStore dataStore) { + if (template == null) { + String msg = String.format("Template Object is not properly initialised %s", this.toString()); + logger.warn(msg); + } imageVO = template; this.dataStore = dataStore; } @@ -98,6 +102,10 @@ public class TemplateObject implements TemplateInfo { } public VMTemplateVO getImage() { + if (imageVO == null) { + String msg = String.format("Template Object is not properly initialised %s", this.toString()); + logger.error(msg); + } // somehow the nullpointer is needed : refacter needed!?! return imageVO; } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index b8f90e46538..c6d9fab5f17 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -42,7 +42,9 @@ import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StorageService; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -54,7 +56,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import javax.inject.Inject; + import java.util.List; +import java.util.Map; public class DefaultHostListener implements HypervisorHostListener { protected Logger logger = LogManager.getLogger(getClass()); @@ -126,7 +130,9 @@ public class DefaultHostListener implements HypervisorHostListener { @Override public boolean hostConnect(long hostId, long poolId) throws StorageConflictException { StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary); - ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool); + Pair, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null); + + ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first()); cmd.setWait(modifyStoragePoolCommandWait); logger.debug(String.format("Sending modify storage pool command to agent: %d for storage pool: %d with timeout %d seconds", hostId, poolId, cmd.getWait())); @@ -139,7 +145,7 @@ public class DefaultHostListener implements HypervisorHostListener { if (!answer.getResult()) { String msg = "Unable to attach storage pool" + poolId + " to the host" + hostId; alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg); - throw new CloudRuntimeException("Unable establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() + + throw new CloudRuntimeException("Unable to establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() + pool.getId()); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 136134698a9..9a3319f79a3 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -505,7 +505,9 @@ public class VolumeServiceImpl implements VolumeService { _snapshotStoreDao.remove(snapStoreVo.getId()); } } else { - _snapshotStoreDao.remove(snapStoreVo.getId()); + if (!StoragePoolType.StorPool.equals(storagePoolVO.getPoolType())) { + _snapshotStoreDao.remove(snapStoreVo.getId()); + } } } snapshotApiService.markVolumeSnapshotsAsDestroyed(vo); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 8a193c1ce80..a92029808d8 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -346,7 +346,7 @@ public class VeeamClient { String type = pair.second(); String path = url.replace(apiURI.toString(), ""); if (type.equals("RestoreSession")) { - return checkIfRestoreSessionFinished(type, path); + checkIfRestoreSessionFinished(type, path); } } return true; @@ -362,17 +362,29 @@ public class VeeamClient { return false; } - protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException { - for (int j = 0; j < this.restoreTimeout; j++) { + + /** + * Checks the status of the restore session. Checked states are "Success" and "Failure".
+ * There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,
+ * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}. + */ + protected void checkIfRestoreSessionFinished(String type, String path) throws IOException { + for (int j = 0; j < restoreTimeout; j++) { HttpResponse relatedResponse = get(path); RestoreSession session = parseRestoreSessionResponse(relatedResponse); if (session.getResult().equals("Success")) { - return true; + return; } + if (session.getResult().equalsIgnoreCase("Failed")) { String sessionUid = session.getUid(); + LOG.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].", + sessionUid, session.getVmDisplayName(), + getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":")))); throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); } + LOG.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout)); + try { Thread.sleep(1000); } catch (InterruptedException ignored) { @@ -931,6 +943,29 @@ public class VeeamClient { return new Pair<>(result.first(), restoreLocation); } + /** + * Tries to retrieve the error's description of the Veeam restore task that resulted in an error. + * @param uid Session uid in Veeam of the restore process; + * @return the description found in Veeam about the cause of error in the restore process. + */ + protected String getRestoreVmErrorDescription(String uid) { + LOG.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid)); + List cmds = Arrays.asList( + String.format("$restoreUid = '%s'", uid), + "$restore = Get-VBRRestoreSession -Id $restoreUid", + "if ($restore) {", + "Write-Output $restore.Description", + "} else {", + "Write-Output 'Cannot find restore session with provided uid $restoreUid'", + "}" + ); + Pair result = executePowerShellCommands(cmds); + if (result != null && result.first()) { + return result.second(); + } + return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid); + } + private boolean isLegacyServer() { return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11); } diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index b00455968c6..b863955ba45 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -59,6 +59,8 @@ public class VeeamClientTest { private VeeamClient mockClient; private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private VeeamClient mock = Mockito.mock(VeeamClient.class); + @Rule public WireMockRule wireMockRule = new WireMockRule(9399); @@ -163,7 +165,7 @@ public class VeeamClientTest { Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse); Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession); Mockito.when(restoreSession.getResult()).thenReturn("No Success"); - Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod(); + Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any")); mockClient.checkIfRestoreSessionFinished("RestoreTest", "any"); fail(); } catch (Exception e) { @@ -172,6 +174,42 @@ public class VeeamClientTest { Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); } + @Test + public void getRestoreVmErrorDescriptionTestFindErrorDescription() { + Pair response = new Pair<>(true, "Example of error description found in Veeam."); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Example of error description found in Veeam.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { + Pair response = new Pair<>(true, "Cannot find restore session with provided uid uuid"); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Cannot find restore session with provided uid uuid", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { + Pair response = new Pair<>(false, null); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + private void verifyBackupMetrics(Map metrics) { Assert.assertEquals(2, metrics.size()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 5cffa77c297..dec7f70e62f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import java.io.BufferedReader; @@ -306,6 +307,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String TUNGSTEN_PATH = "scripts/vm/network/tungsten"; + public static final String INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD = "virt-v2v --version"; + // virt-v2v --version => sample output: virt-v2v 1.42.0rhel=8,release=22.module+el8.10.0+1590+a67ab969 + public static final String OVF_EXPORT_SUPPORTED_CHECK_CMD = "ovftool --version"; + // ovftool --version => sample output: VMware ovftool 4.6.0 (build-21452615) + public static final String OVF_EXPORT_TOOl_GET_VERSION_CMD = "ovftool --version | awk '{print $3}'"; + + public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; + public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; + public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; + private String modifyVlanPath; private String versionStringPath; private String patchScriptPath; @@ -3647,6 +3658,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); cmd.setHostTags(getHostTags()); + cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion())); HealthCheckResult healthCheckResult = getHostHealthCheckResult(); if (healthCheckResult != HealthCheckResult.IGNORE) { cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); @@ -5163,6 +5175,48 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return false; } + public boolean hostSupportsInstanceConversion() { + int exitValue = Script.runSimpleBashScriptForExitValue(INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); + if (isUbuntuHost() && exitValue == 0) { + exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_NBDKIT_PKG_CHECK_CMD); + } + return exitValue == 0; + } + + public boolean hostSupportsWindowsGuestConversion() { + if (isUbuntuHost()) { + int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + int exitValue = Script.runSimpleBashScriptForExitValue(WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + + public boolean hostSupportsOvfExport() { + int exitValue = Script.runSimpleBashScriptForExitValue(OVF_EXPORT_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + + public boolean ovfExportToolSupportsParallelThreads() { + String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + if (StringUtils.isBlank(ovfExportToolVersion)) { + return false; + } + String[] ovfExportToolVersions = ovfExportToolVersion.trim().split("\\."); + if (ovfExportToolVersions.length > 1) { + try { + int majorVersion = Integer.parseInt(ovfExportToolVersions[0]); + int minorVersion = Integer.parseInt(ovfExportToolVersions[1]); + //ovftool version >= 4.4 supports parallel threads + if (majorVersion > 4 || (majorVersion == 4 && minorVersion >= 4)) { + return true; + } + } catch (NumberFormatException ignored) { + } + } + return false; + } + protected void setCpuTopology(CpuModeDef cmd, int vCpusInDef, Map details) { if (!enableManuallySettingCpuTopologyOnKvmVm) { LOGGER.debug(String.format("Skipping manually setting CPU topology on VM's XML due to it is disabled in agent.properties {\"property\": \"%s\", \"value\": %s}.", diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java index ff44c8df2fa..09ee45d5908 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java @@ -16,6 +16,12 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; + public class LibvirtStoragePoolDef { public enum PoolType { ISCSI("iscsi"), NETFS("netfs"), loggerICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); @@ -55,6 +61,7 @@ public class LibvirtStoragePoolDef { private String _authUsername; private AuthenticationType _authType; private String _secretUuid; + private Set _nfsMountOpts = new HashSet<>(); public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, int port, String dir, String targetPath) { _poolType = type; @@ -75,6 +82,15 @@ public class LibvirtStoragePoolDef { _targetPath = targetPath; } + public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, String dir, String targetPath, List nfsMountOpts) { + this(type, poolName, uuid, host, dir, targetPath); + if (CollectionUtils.isNotEmpty(nfsMountOpts)) { + for (String nfsMountOpt : nfsMountOpts) { + this._nfsMountOpts.add(nfsMountOpt); + } + } + } + public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String sourceHost, int sourcePort, String dir, String authUsername, AuthenticationType authType, String secretUuid) { _poolType = type; @@ -124,69 +140,98 @@ public class LibvirtStoragePoolDef { return _authType; } + public Set getNfsMountOpts() { + return _nfsMountOpts; + } + @Override public String toString() { StringBuilder storagePoolBuilder = new StringBuilder(); - if (_poolType == PoolType.GLUSTERFS) { - /* libvirt mounts a Gluster volume, similar to NFS */ - storagePoolBuilder.append("\n"); - } else { - storagePoolBuilder.append("\n"); + String poolTypeXML; + switch (_poolType) { + case NETFS: + if (_nfsMountOpts != null) { + poolTypeXML = "netfs' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0"; + } else { + poolTypeXML = _poolType.toString(); + } + break; + case GLUSTERFS: + /* libvirt mounts a Gluster volume, similar to NFS */ + poolTypeXML = "netfs"; + break; + default: + poolTypeXML = _poolType.toString(); } + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("" + _poolName + "\n"); if (_uuid != null) storagePoolBuilder.append("" + _uuid + "\n"); - if (_poolType == PoolType.NETFS) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - } - if (_poolType == PoolType.RBD) { - storagePoolBuilder.append("\n"); - for (String sourceHost : _sourceHost.split(",")) { + + switch (_poolType) { + case NETFS: + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + break; + + case RBD: + storagePoolBuilder.append("\n"); + for (String sourceHost : _sourceHost.split(",")) { + storagePoolBuilder.append("\n"); + } + + storagePoolBuilder.append("" + _sourceDir + "\n"); + if (_authUsername != null) { + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + } + storagePoolBuilder.append("\n"); + break; + + case GLUSTERFS: + storagePoolBuilder.append("\n"); storagePoolBuilder.append("\n"); - } + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + break; + } - storagePoolBuilder.append("" + _sourceDir + "\n"); - if (_authUsername != null) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - } - storagePoolBuilder.append("\n"); - } - if (_poolType == PoolType.GLUSTERFS) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - } if (_poolType != PoolType.RBD && _poolType != PoolType.POWERFLEX) { storagePoolBuilder.append("\n"); storagePoolBuilder.append("" + _targetPath + "\n"); storagePoolBuilder.append("\n"); } + if (_poolType == PoolType.NETFS && _nfsMountOpts != null) { + storagePoolBuilder.append("\n"); + for (String options : _nfsMountOpts) { + storagePoolBuilder.append("\n"); + } + storagePoolBuilder.append("\n"); + } storagePoolBuilder.append("\n"); return storagePoolBuilder.toString(); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java index 30616e04798..430e4ef851f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java @@ -38,6 +38,19 @@ import org.xml.sax.SAXException; public class LibvirtStoragePoolXMLParser { protected Logger logger = LogManager.getLogger(getClass()); + private List getNFSMountOptsFromRootElement(Element rootElement) { + List nfsMountOpts = new ArrayList<>(); + Element mountOpts = (Element) rootElement.getElementsByTagName("fs:mount_opts").item(0); + if (mountOpts != null) { + NodeList options = mountOpts.getElementsByTagName("fs:option"); + for (int i = 0; i < options.getLength(); i++) { + Element option = (Element) options.item(i); + nfsMountOpts.add(option.getAttribute("name")); + } + } + return nfsMountOpts; + } + public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) { DocumentBuilder builder; try { @@ -95,11 +108,15 @@ public class LibvirtStoragePoolXMLParser { poolName, uuid, host, port, path, targetPath); } else { String path = getAttrValue("dir", "path", source); - Element target = (Element)rootElement.getElementsByTagName("target").item(0); String targetPath = getTagValue("path", target); - return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath); + if (type.equalsIgnoreCase("netfs")) { + List nfsMountOpts = getNFSMountOptsFromRootElement(rootElement); + return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath, nfsMountOpts); + } else { + return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath); + } } } catch (ParserConfigurationException e) { logger.debug(e.toString()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java new file mode 100644 index 00000000000..d94fddeeb44 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -0,0 +1,57 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckConvertInstanceAnswer; +import com.cloud.agent.api.CheckConvertInstanceCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +@ResourceWrapper(handles = CheckConvertInstanceCommand.class) +public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtCheckConvertInstanceCommandWrapper.class); + + @Override + public Answer execute(CheckConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { + if (!serverResource.hostSupportsInstanceConversion()) { + String msg = String.format("Cannot convert the instance from VMware as the virt-v2v binary is not found on host %s. " + + "Please install virt-v2v%s on the host before attempting the instance conversion.", serverResource.getPrivateIp(), serverResource.isUbuntuHost()? ", nbdkit" : ""); + s_logger.info(msg); + return new CheckConvertInstanceAnswer(cmd, false, msg); + } + + if (cmd.getCheckWindowsGuestConversionSupport() && !serverResource.hostSupportsWindowsGuestConversion()) { + String msg = String.format("Cannot convert the instance from VMware as the virtio-win package is not found on host %s. " + + "Please install virtio-win package on the host before attempting the windows guest instance conversion.", serverResource.getPrivateIp()); + s_logger.info(msg); + return new CheckConvertInstanceAnswer(cmd, false, msg); + } + + if (serverResource.hostSupportsOvfExport()) { + return new CheckConvertInstanceAnswer(cmd, true, true, ""); + } + + return new CheckConvertInstanceAnswer(cmd, true, ""); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index bd6634c83a4..4c80a99cb84 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -62,8 +62,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); - protected static final String checkIfConversionIsSupportedCommand = "which virt-v2v"; - @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); @@ -74,9 +72,9 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, timeout); + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + logger.info(String.format("Attempting to convert the OVF %s of the instance %s from %s to KVM", ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); + final String temporaryConvertUuid = UUID.randomUUID().toString(); boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); try { - boolean result = performInstanceConversion(convertInstanceUrl, sourceInstanceName, temporaryPasswordFilePath, - temporaryConvertPath, temporaryConvertUuid, timeout, verboseModeEnabled); + boolean result = performInstanceConversion(sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled); if (!result) { - String err = String.format("The virt-v2v conversion of the instance %s failed. " + - "Please check the agent logs for the virt-v2v output", sourceInstanceName); + String err = String.format("The virt-v2v conversion for the OVF %s failed. " + + "Please check the agent logs for the virt-v2v output", ovfTemplateDirOnConversionLocation); logger.error(err); return new ConvertInstanceAnswer(cmd, false, err); } @@ -130,8 +158,11 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { List disksDefs = xmlParser.getDisks(); disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && @@ -204,11 +256,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper disks) { for (LibvirtVMDef.DiskDef disk : disks) { String[] diskPathParts = disk.getDiskPath().split("/"); @@ -234,6 +281,11 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper 1) { + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } } return new Pair<>(sourceHostIp, sourcePath); } - protected boolean performInstanceConversion(String convertInstanceUrl, String sourceInstanceName, - String temporaryPasswordFilePath, - String temporaryConvertFolder, - String temporaryConvertUuid, - long timeout, boolean verboseModeEnabled) { + private boolean exportOVAFromVMOnVcenter(String vmExportUrl, + String targetOvfDir, + int noOfThreads, + long timeout) { + Script script = new Script("ovftool", timeout, logger); + script.add("--noSSLVerify"); + if (noOfThreads > 1) { + script.add(String.format("--parallelThreads=%s", noOfThreads)); + } + script.add(vmExportUrl); + script.add(targetOvfDir); + + String logPrefix = "export ovf"; + OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix); + script.execute(outputLogger); + int exitValue = script.getExitValue(); + return exitValue == 0; + } + + protected boolean performInstanceConversion(String sourceOVFDirPath, + String temporaryConvertFolder, + String temporaryConvertUuid, + long timeout, boolean verboseModeEnabled) { Script script = new Script("virt-v2v", timeout, logger); script.add("--root", "first"); - script.add("-ic", convertInstanceUrl); - script.add(sourceInstanceName); - script.add("--password-file", temporaryPasswordFilePath); + script.add("-i", "ova"); + script.add(sourceOVFDirPath); script.add("-o", "local"); script.add("-os", temporaryConvertFolder); script.add("-of", "qcow2"); @@ -332,44 +404,13 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper %s", password, passwordFile)); - return passwordFile; - } - - private String getConvertInstanceUrl(RemoteInstanceTO sourceInstance) { - String url = null; - if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - url = getConvertInstanceUrlFromVmware(sourceInstance); - } - return url; - } - - private String getConvertInstanceUrlFromVmware(RemoteInstanceTO vmwareInstance) { - String vcenter = vmwareInstance.getVcenterHost(); - String datacenter = vmwareInstance.getDatacenterName(); - String username = vmwareInstance.getVcenterUsername(); - String host = vmwareInstance.getHostName(); - String cluster = vmwareInstance.getClusterName(); - - String encodedUsername = encodeUsername(username); - return String.format("vpx://%s@%s/%s/%s/%s?no_verify=1", - encodedUsername, vcenter, datacenter, cluster, host); - } protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { String xmlPath = String.format("%s.xml", installPath); if (!new File(xmlPath).exists()) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 97a4c4dc044..6b9e52c3e26 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.cryptsetup.KeyFile; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; @@ -34,6 +35,7 @@ import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.commons.collections.CollectionUtils; import org.libvirt.Connect; import org.libvirt.LibvirtException; import org.libvirt.Secret; @@ -283,9 +285,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } - private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path) throws LibvirtException { + private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List nfsMountOpts) throws LibvirtException { String targetPath = _mountPoint + File.separator + uuid; - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts); _storageLayer.mkdir(targetPath); StoragePool sp = null; try { @@ -374,6 +376,42 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } + private List getNFSMountOptsFromDetails(StoragePoolType type, Map details) { + List nfsMountOpts = null; + if (!type.equals(StoragePoolType.NetworkFilesystem) || details == null) { + return nfsMountOpts; + } + if (details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) { + nfsMountOpts = Arrays.asList(details.get(ApiConstants.NFS_MOUNT_OPTIONS).replaceAll("\\s", "").split(",")); + } + return nfsMountOpts; + } + + private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connect conn, List nfsMountOpts) { + try { + LibvirtStoragePoolDef poolDef = getStoragePoolDef(conn, sp); + Set poolNfsMountOpts = poolDef.getNfsMountOpts(); + boolean mountOptsDiffer = false; + if (poolNfsMountOpts.size() != nfsMountOpts.size()) { + mountOptsDiffer = true; + } else { + for (String nfsMountOpt : nfsMountOpts) { + if (!poolNfsMountOpts.contains(nfsMountOpt)) { + mountOptsDiffer = true; + break; + } + } + } + if (mountOptsDiffer) { + sp.destroy(); + return true; + } + } catch (LibvirtException e) { + logger.error("Failure in destroying the pre-existing storage pool for changing the NFS mount options" + e); + } + return false; + } + private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) { LibvirtStoragePoolDef spd; @@ -671,12 +709,21 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } catch (LibvirtException e) { logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e); } + } + + List nfsMountOpts = getNFSMountOptsFromDetails(type, details); + if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) && + destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { + sp = null; + } + + if (sp == null) { logger.debug("Attempting to create storage pool " + name); if (type == StoragePoolType.NetworkFilesystem) { try { - sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path); + sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts); } catch (LibvirtException e) { logger.error("Failed to create netfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); @@ -684,7 +731,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } else if (type == StoragePoolType.Gluster) { try { - sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path); + sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null); } catch (LibvirtException e) { logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java index 51a47e9a025..712b38b0bb4 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java @@ -19,6 +19,9 @@ package com.cloud.hypervisor.kvm.resource; +import java.util.ArrayList; +import java.util.List; + import junit.framework.TestCase; import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.PoolType; import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.AuthenticationType; @@ -47,6 +50,14 @@ public class LibvirtStoragePoolDefTest extends TestCase { assertEquals(port, pool.getSourcePort()); assertEquals(dir, pool.getSourceDir()); assertEquals(targetPath, pool.getTargetPath()); + + List nfsMountOpts = new ArrayList<>(); + nfsMountOpts.add("vers=4.1"); + nfsMountOpts.add("nconnect=4"); + pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); + assertTrue(pool.getNfsMountOpts().contains("vers=4.1")); + assertTrue(pool.getNfsMountOpts().contains("nconnect=4")); + assertEquals(pool.getNfsMountOpts().size(), 2); } @Test @@ -57,12 +68,38 @@ public class LibvirtStoragePoolDefTest extends TestCase { String host = "127.0.0.1"; String dir = "/export/primary"; String targetPath = "/mnt/" + uuid; + List nfsMountOpts = new ArrayList<>(); + nfsMountOpts.add("vers=4.1"); + nfsMountOpts.add("nconnect=4"); - LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath); + LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); - String expectedXml = "\n" + name + "\n" + uuid + "\n" + + String expectedXml = "\n" + + "" +name + "\n" + uuid + "\n" + "\n\n\n\n\n" + - "" + targetPath + "\n\n\n"; + "" + targetPath + "\n\n" + + "\n\n\n\n\n"; + + assertEquals(expectedXml, pool.toString()); + } + + @Test + public void testGlusterFSStoragePool() { + PoolType type = PoolType.GLUSTERFS; + String name = "myGFSPool"; + String uuid = "89a605bc-d470-4637-b3df-27388be452f5"; + String host = "127.0.0.1"; + String dir = "/export/primary"; + String targetPath = "/mnt/" + uuid; + List nfsMountOpts = new ArrayList<>(); + + LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); + + String expectedXml = "\n" + + "" +name + "\n" + uuid + "\n" + + "\n\n\n" + + "\n\n\n" + + "" + targetPath + "\n\n\n"; assertEquals(expectedXml, pool.toString()); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java index 3637b7b1f9b..5854c21186f 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java @@ -30,7 +30,7 @@ public class LibvirtStoragePoolXMLParserTest extends TestCase { @Test public void testParseNfsStoragePoolXML() { - String poolXML = "\n" + + String poolXML = "\n" + " feff06b5-84b2-3258-b5f9-1953217295de\n" + " feff06b5-84b2-3258-b5f9-1953217295de\n" + " 111111111\n" + @@ -49,12 +49,18 @@ public class LibvirtStoragePoolXMLParserTest extends TestCase { " 0\n" + " \n" + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + ""; LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser(); LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML); - Assert.assertEquals("10.11.12.13", pool.getSourceHost()); + assertEquals("10.11.12.13", pool.getSourceHost()); + assertTrue(pool.getNfsMountOpts().contains("vers=4.1")); + assertTrue(pool.getNfsMountOpts().contains("nconnect=8")); } @Test diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java new file mode 100644 index 00000000000..3cad9c27a68 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java @@ -0,0 +1,67 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckConvertInstanceCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckConvertInstanceCommandWrapperTest { + + @Spy + private LibvirtCheckConvertInstanceCommandWrapper checkConvertInstanceCommandWrapper = Mockito.spy(LibvirtCheckConvertInstanceCommandWrapper.class); + + @Mock + private LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + CheckConvertInstanceCommand checkConvertInstanceCommandMock; + + @Before + public void setUp() { + } + + @Test + public void testCheckInstanceCommand_success() { + Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true); + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + assertTrue(answer.getResult()); + } + + @Test + public void testCheckInstanceCommand_failure() { + Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(false); + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + assertFalse(answer.getResult()); + assertTrue(StringUtils.isNotBlank(answer.getDetails())); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index b9316c6c042..79cd55763db 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -71,12 +71,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { private static final String secondaryPoolUrl = "nfs://192.168.1.1/secondary"; private static final String vmName = "VmToImport"; - private static final String hostName = "VmwareHost1"; - private static final String vmwareVcenter = "192.168.1.2"; - private static final String vmwareDatacenter = "Datacenter"; - private static final String vmwareCluster = "Cluster"; - private static final String vmwareUsername = "administrator@vsphere.local"; - private static final String vmwarePassword = "password"; @Before public void setUp() { @@ -88,15 +82,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); } - @Test - public void testIsInstanceConversionSupportedOnHost() { - try (MockedStatic + + diff --git a/ui/src/views/infra/network/IpRangesTabPublic.vue b/ui/src/views/infra/network/IpRangesTabPublic.vue index 40a3140c35f..3955b8c0359 100644 --- a/ui/src/views/infra/network/IpRangesTabPublic.vue +++ b/ui/src/views/infra/network/IpRangesTabPublic.vue @@ -246,35 +246,42 @@
-
{{ $t('label.set.reservation') }}
+ +
-
- + + +
- - - - - - - {{ domain.path || domain.name || domain.description }} - - - - +
+ + + + +
+ + + + + {{ domain.path || domain.name || domain.description }} + + + + +
@@ -340,11 +347,13 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import TooltipButton from '@/components/widgets/TooltipButton' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'IpRangesTabPublic', components: { - TooltipButton + TooltipButton, + TooltipLabel }, props: { resource: { @@ -712,6 +721,10 @@ export default { } } + .tooltip-label-wrapper { + color: rgba(0, 0, 0, 0.85); + } + .ant-list-item { padding-top: 0; padding-bottom: 0; diff --git a/ui/src/views/infra/zone/ZoneWizard.vue b/ui/src/views/infra/zone/ZoneWizard.vue index 0f4c7f7ad31..9211aed92a5 100644 --- a/ui/src/views/infra/zone/ZoneWizard.vue +++ b/ui/src/views/infra/zone/ZoneWizard.vue @@ -232,7 +232,7 @@ export default { width: 100%; @media (min-width: 1000px) { - width: 800px; + width: 1000px; } :deep(.form-action) { diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue index 24ddd264486..ef2cba1c3d9 100644 --- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue +++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue @@ -530,6 +530,15 @@ export default { primaryStorageProtocol: ['vmfs', 'datastorecluster'] } }, + { + title: 'label.nfsmountopts', + key: 'primaryStorageNFSMountOptions', + required: false, + display: { + primaryStorageProtocol: 'nfs', + hypervisor: ['KVM', 'Simulator'] + } + }, { title: 'label.resourcegroup', key: 'primaryStorageLinstorResourceGroup', diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index 4c40dfef2ed..e562fb33d72 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -1439,6 +1439,7 @@ export default { path = '/' + path } url = this.nfsURL(server, path) + params['details[0].nfsmountopts'] = this.prefillContent.primaryStorageNFSMountOptions } else if (protocol === 'SMB') { let path = this.prefillContent?.primaryStoragePath || '' if (path.substring(0, 1) !== '/') { @@ -2145,7 +2146,11 @@ export default { resolve() }).catch(error => { message = error.response.headers['x-description'] - reject(message) + if (message.includes('is already in the database')) { + resolve() + } else { + reject(message) + } }) }) }, diff --git a/ui/src/views/infra/zone/ZoneWizardZoneDetailsStep.vue b/ui/src/views/infra/zone/ZoneWizardZoneDetailsStep.vue index 8aea3ffa17a..05c5adc1bd2 100644 --- a/ui/src/views/infra/zone/ZoneWizardZoneDetailsStep.vue +++ b/ui/src/views/infra/zone/ZoneWizardZoneDetailsStep.vue @@ -158,7 +158,7 @@ @@ -166,7 +166,7 @@ diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index 4fb559b2628..66efb764f0c 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -158,7 +158,7 @@ v-if="cluster.hypervisortype === 'KVM' && selectedVmwareVcenter" :resourceKey="cluster.id" :selectOptions="kvmHostsForConversion" - :checkBoxLabel="'(Optional) Select a KVM host in the cluster to perform the instance conversion through virt-v2v'" + :checkBoxLabel="$t('message.select.kvm.host.instance.conversion')" :defaultCheckBoxValue="false" :reversed="false" @handle-checkselectpair-change="updateSelectedKvmHostForConversion" @@ -167,11 +167,11 @@ + + + +