From 371ad9f55b3592f652edbbe22dd8e13b71f524da Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 7 Dec 2023 04:29:56 -0300 Subject: [PATCH] New Feature: Import VMware VMs into KVM (#7881) This PR adds the capability in CloudStack to convert VMware Instances disk(s) to KVM using virt-v2v and import them as CloudStack instances. It enables CloudStack operators to import VMware instances from vSphere into a KVM cluster managed by CloudStack. vSphere/VMware setup might be managed by CloudStack or be a standalone setup. CloudStack will let the administrator select a VM from an existing VMware vCenter in the CloudStack environment or external vCenter requesting vCenter IP, Datacenter name and credentials. The migrated VM will be imported as a KVM instance The migration is done through virt-v2v: https://access.redhat.com/articles/1351473, https://www.ovirt.org/develop/release-management/features/virt/virt-v2v-integration.html The migration process timeout can be set by the setting convert.instance.process.timeout Before attempting the virt-v2v migration, CloudStack will create a clone of the source VM on VMware. The clone VM will be removed after the registration process finishes. CloudStack will delegate the migration action to a KVM host and the host will attempt to migrate the VM invoking virt-v2v. In case the guest OS is not supported then CloudStack will handle the error operation as a failure The migration process using virt-v2v may not be a fast process CloudStack will not perform any check about the guest OS compatibility for the virt-v2v library as indicated on: https://access.redhat.com/articles/1351473. --- agent/conf/agent.properties | 3 + .../agent/properties/AgentProperties.java | 7 + .../cloud/agent/api/to/RemoteInstanceTO.java | 88 +++ .../java/com/cloud/dc}/VmwareDatacenter.java | 2 +- .../com/cloud/hypervisor/HypervisorGuru.java | 22 + .../java/com/cloud/vm/VmDetailConstants.java | 12 + .../apache/cloudstack/api/ApiConstants.java | 6 + .../cloudstack/api/ResponseGenerator.java | 4 + .../admin/vm/ImportUnmanagedInstanceCmd.java | 7 +- .../api/command/admin/vm/ImportVmCmd.java | 180 +++++ .../response/UnmanagedInstanceResponse.java | 12 + .../response/VmwareDatacenterResponse.java | 2 +- .../cloudstack/vm/UnmanagedInstanceTO.java | 30 + .../apache/cloudstack/vm/VmImportService.java | 12 + .../agent/api/ConvertInstanceAnswer.java | 40 + .../agent/api/ConvertInstanceCommand.java | 63 ++ debian/control | 2 +- .../com/cloud/storage/StorageManager.java | 8 + .../orchestration/NetworkOrchestrator.java | 30 +- .../com/cloud/dc}/VmwareDatacenterVO.java | 2 +- .../cloud/dc}/dao/VmwareDatacenterDao.java | 4 +- .../vmware/dao/VmwareDatacenterDaoImpl.java | 3 +- ...spring-engine-schema-core-daos-context.xml | 1 + .../cloudstack/sioc/SiocManagerImpl.java | 4 +- .../backup/VeeamBackupProvider.java | 4 +- .../resource/LibvirtComputingResource.java | 7 + .../hypervisor/kvm/resource/LibvirtVMDef.java | 14 +- .../LibvirtConvertInstanceCommandWrapper.java | 400 ++++++++++ ...virtConvertInstanceCommandWrapperTest.java | 310 ++++++++ .../com/cloud/hypervisor/guru/VMwareGuru.java | 125 ++- .../vmware/VmwareDatacenterService.java | 6 + .../vmware/VmwareServerDiscoverer.java | 3 +- .../vmware/manager/VmwareManagerImpl.java | 66 +- .../vmware/resource/VmwareResource.java | 286 +------ .../resource/VmwareStorageProcessor.java | 2 +- .../command/admin/zone/AddVmwareDcCmd.java | 2 +- .../admin/zone/ListVmwareDcVmsCmd.java | 139 ++++ .../command/admin/zone/ListVmwareDcsCmd.java | 2 +- .../command/admin/zone/UpdateVmwareDcCmd.java | 2 +- .../core/spring-vmware-core-context.xml | 2 - .../vmware/VmwareDatacenterApiUnitTest.java | 3 +- .../vmware/manager/VmwareManagerImplTest.java | 6 +- .../java/com/cloud/api/ApiResponseHelper.java | 69 ++ .../cloud/hypervisor/HypervisorGuruBase.java | 12 + .../com/cloud/storage/StorageManagerImpl.java | 3 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 31 +- .../vm/UnmanagedVMsManagerImpl.java | 712 ++++++++++++++---- .../com/cloud/api/ApiResponseHelperTest.java | 29 + .../com/cloud/vm/UserVmManagerImplTest.java | 21 + .../vm/UnmanagedVMsManagerImplTest.java | 327 +++++++- tools/apidoc/gen_toc.py | 2 + ui/public/locales/en.json | 17 +- ui/src/components/widgets/TooltipLabel.vue | 1 + ui/src/views/compute/DeployVM.vue | 3 + .../wizard/ComputeOfferingSelection.vue | 8 + .../compute/wizard/MultiNetworkSelection.vue | 34 +- .../compute/wizard/NetworkConfiguration.vue | 5 + .../views/image/RegisterOrUploadTemplate.vue | 19 +- .../views/tools/ImportUnmanagedInstance.vue | 324 ++++++-- ui/src/views/tools/ManageInstances.vue | 357 +++++++-- ui/src/views/tools/SelectVmwareVcenter.vue | 273 +++++++ .../cloud/utils/script/OutputInterpreter.java | 28 + .../java/com/cloud/utils/script/Script.java | 19 +- vmware-base/pom.xml | 6 + .../hypervisor/vmware/mo/DatacenterMO.java | 22 +- .../hypervisor/vmware/util/VmwareHelper.java | 291 +++++++ .../vmware/util/VmwareHelperTest.java | 51 ++ 67 files changed, 3919 insertions(+), 668 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => api/src/main/java/com/cloud/dc}/VmwareDatacenter.java (96%) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java rename {plugins/hypervisors/vmware => api}/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java (97%) create mode 100644 core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/VmwareDatacenterVO.java (99%) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/dao/VmwareDatacenterDao.java (96%) rename {plugins/hypervisors/vmware => engine/schema}/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java (97%) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java create mode 100644 ui/src/views/tools/SelectVmwareVcenter.vue diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 3f07ba16237..b15534a0be6 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -419,3 +419,6 @@ iscsi.session.cleanup.enabled=false # Timeout (in milliseconds) of the KVM heartbeat checker. # kvm.heartbeat.checker.timeout=360000 + +# Instance Conversion from Vmware to KVM through virt-v2v. Enable verbose mode +# virtv2v.verbose.enabled=false diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 8f51d470f07..c682314097a 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -733,6 +733,13 @@ public class AgentProperties{ */ public static final Property IOTHREADS = new Property<>("iothreads", 1); + /** + * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM + * Data type: Boolean.
+ * Default value: false + */ + public static final Property VIRTV2V_VERBOSE_ENABLED = new Property<>("virtv2v.verbose.enabled", false); + /** * BGP controll CIDR * Data type: String.
diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java new file mode 100644 index 00000000000..6e7aa8b21e2 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api.to; + +import com.cloud.agent.api.LogLevel; +import com.cloud.hypervisor.Hypervisor; + +import java.io.Serializable; + +public class RemoteInstanceTO implements Serializable { + + private Hypervisor.HypervisorType hypervisorType; + private String hostName; + private String instanceName; + + // Vmware Remote Instances parameters + // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes + private String vcenterUsername; + @LogLevel(LogLevel.Log4jLevel.Off) + private String vcenterPassword; + private String vcenterHost; + private String datacenterName; + private String clusterName; + + public RemoteInstanceTO() { + } + + public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost, + String datacenterName, String clusterName, + String vcenterUsername, String vcenterPassword) { + this.hypervisorType = Hypervisor.HypervisorType.VMware; + this.hostName = hostName; + this.instanceName = instanceName; + this.vcenterHost = vcenterHost; + this.datacenterName = datacenterName; + this.clusterName = clusterName; + this.vcenterUsername = vcenterUsername; + this.vcenterPassword = vcenterPassword; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return this.hypervisorType; + } + + public String getInstanceName() { + return this.instanceName; + } + + public String getHostName() { + return this.hostName; + } + + public String getVcenterUsername() { + return vcenterUsername; + } + + public String getVcenterPassword() { + return vcenterPassword; + } + + public String getVcenterHost() { + return vcenterHost; + } + + public String getDatacenterName() { + return datacenterName; + } + + public String getClusterName() { + return clusterName; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java rename to api/src/main/java/com/cloud/dc/VmwareDatacenter.java index b1f233c3606..859ff190b65 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java +++ b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 2dfa707b57b..3c7dbac6442 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -33,6 +33,7 @@ import com.cloud.utils.component.Adapter; import com.cloud.vm.NicProfile; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; public interface HypervisorGuru extends Adapter { @@ -104,4 +105,25 @@ public interface HypervisorGuru extends Adapter { * @return a list of commands to perform for a successful migration */ List finalizeMigrate(VirtualMachine vm, Map volumeToPool); + + + /** + * Will perform a clone of a VM on an external host (if the guru can handle) + * @param hostIp VM's source host IP + * @param vmName name of the source VM to clone from + * @param params hypervisor specific additional parameters + * @return a reference to the cloned VM + */ + UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, + Map params); + + /** + * Removes a VM created as a clone of a VM on an external host + * @param hostIp VM's source host IP + * @param vmName name of the VM to remove + * @param params hypervisor specific additional parameters + * @return true if the operation succeeds, false if not + */ + boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, + Map params); } diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 83c7529b22b..124d9d50d5b 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -86,4 +86,16 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + + // VMware to KVM VM migrations specific + String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; + String VMWARE_VCENTER_HOST = String.format("%s-vcenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_DATACENTER_NAME = String.format("%s-datacenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_CLUSTER_NAME = String.format("%s-cluster", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_USERNAME = String.format("%s-username", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_PASSWORD = String.format("%s-password", VMWARE_TO_KVM_PREFIX); + String VMWARE_VM_NAME = String.format("%s-vmname", VMWARE_TO_KVM_PREFIX); + String VMWARE_HOST_NAME = String.format("%s-host", VMWARE_TO_KVM_PREFIX); + String VMWARE_DISK = String.format("%s-disk", VMWARE_TO_KVM_PREFIX); + String VMWARE_MAC_ADDRESSES = String.format("%s-mac-addresses", VMWARE_TO_KVM_PREFIX); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ff188684557..6151b6f5945 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String CERTIFICATE_SERIALNUM = "serialnum"; public static final String CERTIFICATE_SUBJECT = "subject"; public static final String CERTIFICATE_VALIDITY = "validity"; + public static final String CONVERT_INSTANCE_HOST_ID = "convertinstancehostid"; + public static final String CONVERT_INSTANCE_STORAGE_POOL_ID = "convertinstancepoolid"; public static final String ENABLED_REVOCATION_CHECK = "enabledrevocationcheck"; public static final String CONTROLLER = "controller"; public static final String CONTROLLER_UNIT = "controllerunit"; @@ -120,6 +122,7 @@ public class ApiConstants { public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; + public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DESCRIPTION = "description"; @@ -207,6 +210,7 @@ public class ApiConstants { public static final String HIDE_IP_ADDRESS_USAGE = "hideipaddressusage"; public static final String HOST_ID = "hostid"; public static final String HOST_IDS = "hostids"; + public static final String HOST_IP = "hostip"; public static final String HOST_NAME = "hostname"; public static final String HOST_CONTROL_STATE = "hostcontrolstate"; public static final String HOSTS_MAP = "hostsmap"; @@ -779,6 +783,7 @@ public class ApiConstants { public static final String VSM_CONFIG_STATE = "vsmconfigstate"; public static final String VSM_DEVICE_STATE = "vsmdevicestate"; public static final String VCENTER = "vcenter"; + public static final String EXISTING_VCENTER_ID = "existingvcenterid"; public static final String ADD_VSM_FLAG = "addvsmflag"; public static final String END_POINT = "endpoint"; public static final String REGION_ID = "regionid"; @@ -1059,6 +1064,7 @@ public class ApiConstants { public static final String SOURCE_NAT_IP = "sourcenatipaddress"; public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid"; public static final String HAS_RULES = "hasrules"; + public static final String IMPORT_SOURCE = "importsource"; public static final String OBJECT_STORAGE = "objectstore"; public static final String HEURISTIC_RULE = "heuristicrule"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 0bddf6d2994..ef759aaf9c3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -123,6 +123,7 @@ import org.apache.cloudstack.api.response.TemplatePermissionsResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.TrafficMonitorResponse; import org.apache.cloudstack.api.response.TrafficTypeResponse; +import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UpgradeRouterTemplateResponse; import org.apache.cloudstack.api.response.UsageRecordResponse; import org.apache.cloudstack.api.response.UserDataResponse; @@ -237,6 +238,7 @@ import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; import com.cloud.vm.snapshot.VMSnapshot; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; public interface ResponseGenerator { UserResponse createUserResponse(UserAccount user); @@ -538,6 +540,8 @@ public interface ResponseGenerator { FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); + UnmanagedInstanceResponse createUnmanagedInstanceResponse(UnmanagedInstanceTO instance, Cluster cluster, Host host); + SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic); IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index 70233317cc5..532a3f0d392 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -124,7 +124,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { type = CommandType.UUID, entityType = ServiceOfferingResponse.class, required = true, - description = "the ID of the service offering for the virtual machine") + description = "the service offering for the virtual machine") private Long serviceOfferingId; @Parameter(name = ApiConstants.NIC_NETWORK_LIST, @@ -154,7 +154,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, - description = "VM is imported despite some of its NIC's MAC addresses are already present") + description = "VM is imported despite some of its NIC's MAC addresses are already present, in case the MAC address exists then a new MAC address is generated") private Boolean forced; ///////////////////////////////////////////////////// @@ -279,7 +279,8 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Override public String getEventDescription() { - return "Importing unmanaged VM"; + String vmName = this.name; + return String.format("Importing unmanaged VM: %s", vmName); } public boolean isForced() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java new file mode 100644 index 00000000000..01f517fb837 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +@APICommand(name = "importVm", + description = "Import virtual machine from a unmanaged host into CloudStack", + responseObject = UserVmResponse.class, + responseView = ResponseObject.ResponseView.Full, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.19.0") +public class ImportVmCmd extends ImportUnmanagedInstanceCmd { + + public static final Logger LOGGER = Logger.getLogger(ImportVmCmd.class); + + @Parameter(name = ApiConstants.HYPERVISOR, + type = CommandType.STRING, + required = true, + description = "hypervisor type of the host") + private String hypervisor; + + @Parameter(name = ApiConstants.IMPORT_SOURCE, + type = CommandType.STRING, + required = true, + description = "Source location for Import" ) + private String importSource; + + // Import from Vmware to KVM migration parameters + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.HOST_IP, + type = BaseCmd.CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.") + private String host; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.") + private String clusterName; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) The Username required to connect to resource.") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) The password for the specified username.") + private String password; + + @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + private Long convertInstanceHostId; + + @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") + private Long convertStoragePoolId; + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_IMPORT; + } + + @Override + public String getEventDescription() { + String vmName = getName(); + if (ObjectUtils.anyNotNull(vcenter, existingVcenterId)) { + String msg = StringUtils.isNotBlank(vcenter) ? + String.format("external vCenter: %s - datacenter: %s", vcenter, datacenterName) : + String.format("existing vCenter Datacenter with ID: %s", existingVcenterId); + return String.format("Importing unmanaged VM: %s from %s - VM: %s", getDisplayName(), msg, vmName); + } + return String.format("Importing unmanaged VM: %s", vmName); + } + + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + public String getHost() { + return host; + } + + public String getVcenter() { + return vcenter; + } + + public String getDatacenterName() { + return datacenterName; + } + + public String getClusterName() { + return clusterName; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public Long getConvertInstanceHostId() { + return convertInstanceHostId; + } + + public Long getConvertStoragePoolId() { + return convertStoragePoolId; + } + + public String getHypervisor() { + return hypervisor; + } + + public String getImportSource() { + return importSource; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + UserVmResponse response = vmImportService.importVm(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index e866b19e1c1..7a26b178591 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -39,6 +39,10 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "the ID of the cluster to which virtual machine belongs") private String clusterId; + @SerializedName(ApiConstants.CLUSTER_NAME) + @Param(description = "the name of the cluster to which virtual machine belongs") + private String clusterName; + @SerializedName(ApiConstants.HOST_ID) @Param(description = "the ID of the host to which virtual machine belongs") private String hostId; @@ -104,6 +108,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { this.clusterId = clusterId; } + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + public String getHostId() { return hostId; } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java similarity index 97% rename from plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java rename to api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java index ddd4a364a0a..3cf06f38242 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java @@ -23,7 +23,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.serializer.Param; @EntityReference(value = VmwareDatacenter.class) diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 95675f2bf34..a4748155b76 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -33,6 +33,8 @@ public class UnmanagedInstanceTO { private PowerState powerState; + private PowerState cloneSourcePowerState; + private Integer cpuCores; private Integer cpuCoresPerSocket; @@ -45,6 +47,10 @@ public class UnmanagedInstanceTO { private String operatingSystem; + private String clusterName; + + private String hostName; + private List disks; private List nics; @@ -73,6 +79,14 @@ public class UnmanagedInstanceTO { this.powerState = powerState; } + public PowerState getCloneSourcePowerState() { + return cloneSourcePowerState; + } + + public void setCloneSourcePowerState(PowerState cloneSourcePowerState) { + this.cloneSourcePowerState = cloneSourcePowerState; + } + public Integer getCpuCores() { return cpuCores; } @@ -121,6 +135,22 @@ public class UnmanagedInstanceTO { this.operatingSystem = operatingSystem; } + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + public List getDisks() { return disks; } diff --git a/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java b/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java index cce28474541..e5b121cd2d6 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java +++ b/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java @@ -18,12 +18,24 @@ package org.apache.cloudstack.vm; import org.apache.cloudstack.api.command.admin.vm.ImportUnmanagedInstanceCmd; +import org.apache.cloudstack.api.command.admin.vm.ImportVmCmd; import org.apache.cloudstack.api.command.admin.vm.ListUnmanagedInstancesCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UserVmResponse; public interface VmImportService { + + enum ImportSource { + UNMANAGED, VMWARE, EXTERNAL, SHARED, LOCAL; + + @Override + public String toString() { + return name().toLowerCase(); + } + } + ListResponse listUnmanagedInstances(ListUnmanagedInstancesCmd cmd); UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd); + UserVmResponse importVm(ImportVmCmd cmd); } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java new file mode 100644 index 00000000000..829888570a6 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +public class ConvertInstanceAnswer extends Answer { + + public ConvertInstanceAnswer() { + super(); + } + private UnmanagedInstanceTO convertedInstance; + + public ConvertInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { + super(command, true, ""); + this.convertedInstance = convertedInstance; + } + + public UnmanagedInstanceTO getConvertedInstance() { + return convertedInstance; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java new file mode 100644 index 00000000000..63234b04480 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; + +import java.util.List; + +public class ConvertInstanceCommand extends Command { + + private RemoteInstanceTO sourceInstance; + private Hypervisor.HypervisorType destinationHypervisorType; + private List destinationStoragePools; + private DataStoreTO conversionTemporaryLocation; + + public ConvertInstanceCommand() { + } + + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, + List destinationStoragePools, DataStoreTO conversionTemporaryLocation) { + this.sourceInstance = sourceInstance; + this.destinationHypervisorType = destinationHypervisorType; + this.destinationStoragePools = destinationStoragePools; + this.conversionTemporaryLocation = conversionTemporaryLocation; + } + + public RemoteInstanceTO getSourceInstance() { + return sourceInstance; + } + + public Hypervisor.HypervisorType getDestinationHypervisorType() { + return destinationHypervisorType; + } + + public List getDestinationStoragePools() { + return destinationStoragePools; + } + + public DataStoreTO getConversionTemporaryLocation() { + return conversionTemporaryLocation; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/debian/control b/debian/control index 7c1ff8e0b1b..9fec540975e 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, aria2, ufw, apparmor +Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 1437457725a..f5cf443211c 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -96,6 +96,14 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); + ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, + "convert.vmware.instance.to.kvm.timeout", + "Storage", + "8", + "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", + true, + ConfigKey.Scope.Global, + null); ConfigKey KvmAutoConvergence = new ConfigKey<>(Boolean.class, "kvm.auto.convergence", "Storage", diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 960dddbe847..30fd35e5f37 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -4588,18 +4587,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public NicVO doInTransaction(TransactionStatus status) { NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddress); + String macAddressToPersist = macAddress; if (existingNic != null) { - if (!forced) { - throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + - " and forced flag is disabled"); - } - s_logger.debug("Removing existing NIC with MAC address = " + macAddress + " on network with ID = " + network.getId()); - existingNic.setState(Nic.State.Deallocating); - existingNic.setRemoved(new Date()); - _nicDao.update(existingNic.getId(), existingNic); + macAddressToPersist = generateNewMacAddressIfForced(network, macAddress, forced); } NicVO vo = new NicVO(network.getGuruName(), vm.getId(), network.getId(), vm.getType()); - vo.setMacAddress(macAddress); + vo.setMacAddress(macAddressToPersist); vo.setAddressFormat(Networks.AddressFormat.Ip4); if (NetUtils.isValidIp4(finalGuestIp) && StringUtils.isNotEmpty(network.getGateway())) { vo.setIPv4Address(finalGuestIp); @@ -4643,6 +4636,23 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new Pair(vmNic, Integer.valueOf(deviceId)); } + private String generateNewMacAddressIfForced(Network network, String macAddress, boolean forced) { + if (!forced) { + throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + + " and forced flag is disabled"); + } + try { + s_logger.debug(String.format("Generating a new mac address on network %s as the mac address %s already exists", network.getName(), macAddress)); + String newMacAddress = _networkModel.getNextAvailableMacAddressInNetwork(network.getId()); + s_logger.debug(String.format("Successfully generated the mac address %s, using it instead of the conflicting address %s", newMacAddress, macAddress)); + return newMacAddress; + } catch (InsufficientAddressCapacityException e) { + String msg = String.format("Could not generate a new mac address on network %s", network.getName()); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + @Override public void unmanageNics(VirtualMachineProfile vm) { if (s_logger.isDebugEnabled()) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java similarity index 99% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java rename to engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java index 86597b2e5c9..6390d923ed8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import java.util.UUID; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java rename to engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java index 2754e91d26c..1ed1f8d5a0f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.dao; +package com.cloud.dc.dao; import java.util.List; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.GenericDao; public interface VmwareDatacenterDao extends GenericDao { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java b/engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java similarity index 97% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java rename to engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java index e44087e6ada..b4bd56f5b4e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java @@ -20,10 +20,11 @@ package com.cloud.hypervisor.vmware.dao; import java.util.List; +import com.cloud.dc.dao.VmwareDatacenterDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index a8c239b8f6b..5d958383161 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -281,6 +281,7 @@ + diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index 966c83722ec..f012dbfa972 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -35,9 +35,9 @@ import org.springframework.stereotype.Component; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.storage.Storage.StoragePoolType; diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 02f08d602bb..c0091e47061 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -41,9 +41,9 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index da769052c45..5d4b29a9b4f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -743,6 +743,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected StorageSubsystemCommandHandler storageHandler; + private boolean convertInstanceVerboseMode = false; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; @@ -803,6 +804,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return networkDirectDevice; } + public boolean isConvertInstanceVerboseModeEnabled() { + return convertInstanceVerboseMode; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -991,6 +996,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv params.putAll(getDeveloperProperties()); } + convertInstanceVerboseMode = BooleanUtils.isTrue(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VIRTV2V_VERBOSE_ENABLED)); + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index aac44fc1419..d31a6ab38db 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -609,7 +609,7 @@ public class LibvirtVMDef { } } - enum DiskType { + public enum DiskType { FILE("file"), BLOCK("block"), DIRECTROY("dir"), NETWORK("network"); String _diskType; @@ -2113,14 +2113,14 @@ public class LibvirtVMDef { private String path = "/dev/random"; private RngModel rngModel = RngModel.VIRTIO; private RngBackendModel rngBackendModel = RngBackendModel.RANDOM; - private int rngRateBytes = 2048; - private int rngRatePeriod = 1000; + private Integer rngRateBytes = 2048; + private Integer rngRatePeriod = 1000; public RngDef(String path) { this.path = path; } - public RngDef(String path, int rngRateBytes, int rngRatePeriod) { + public RngDef(String path, Integer rngRateBytes, Integer rngRatePeriod) { this.path = path; this.rngRateBytes = rngRateBytes; this.rngRatePeriod = rngRatePeriod; @@ -2139,7 +2139,7 @@ public class LibvirtVMDef { this.rngBackendModel = rngBackendModel; } - public RngDef(String path, RngBackendModel rngBackendModel, int rngRateBytes, int rngRatePeriod) { + public RngDef(String path, RngBackendModel rngBackendModel, Integer rngRateBytes, Integer rngRatePeriod) { this.path = path; this.rngBackendModel = rngBackendModel; this.rngRateBytes = rngRateBytes; @@ -2164,11 +2164,11 @@ public class LibvirtVMDef { } public int getRngRateBytes() { - return rngRateBytes; + return rngRateBytes != null ? rngRateBytes : 0; } public int getRngRatePeriod() { - return rngRatePeriod; + return rngRatePeriod != null ? rngRatePeriod : 0; } @Override diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java new file mode 100644 index 00000000000..a26311891c8 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -0,0 +1,400 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ConvertInstanceAnswer; +import com.cloud.agent.api.ConvertInstanceCommand; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@ResourceWrapper(handles = ConvertInstanceCommand.class) +public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtConvertInstanceCommandWrapper.class); + + private static final List supportedInstanceConvertSourceHypervisors = + List.of(Hypervisor.HypervisorType.VMware); + + protected static final String checkIfConversionIsSupportedCommand = "which virt-v2v"; + + @Override + public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType(); + String sourceInstanceName = sourceInstance.getInstanceName(); + Hypervisor.HypervisorType destinationHypervisorType = cmd.getDestinationHypervisorType(); + List destinationStoragePools = cmd.getDestinationStoragePools(); + DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); + long timeout = (long) cmd.getWait() * 1000; + + if (!isInstanceConversionSupportedOnHost()) { + String msg = String.format("Cannot convert the instance %s from VMware as the virt-v2v binary is not found. " + + "Please install virt-v2v on the host before attempting the instance conversion", sourceInstanceName); + s_logger.info(msg); + return new ConvertInstanceAnswer(cmd, false, msg); + } + + if (!areSourceAndDestinationHypervisorsSupported(sourceHypervisorType, destinationHypervisorType)) { + String err = destinationHypervisorType != Hypervisor.HypervisorType.KVM ? + String.format("The destination hypervisor type is %s, KVM was expected, cannot handle it", destinationHypervisorType) : + String.format("The source hypervisor type %s is not supported for KVM conversion", sourceHypervisorType); + s_logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool temporaryStoragePool = getTemporaryStoragePool(conversionTemporaryLocation, storagePoolMgr); + + s_logger.info(String.format("Attempting to convert the instance %s from %s to KVM", + sourceInstanceName, sourceHypervisorType)); + final String convertInstanceUrl = getConvertInstanceUrl(sourceInstance); + final String temporaryConvertUuid = UUID.randomUUID().toString(); + final String temporaryPasswordFilePath = createTemporaryPasswordFileAndRetrievePath(sourceInstance); + final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); + + try { + boolean result = performInstanceConversion(convertInstanceUrl, sourceInstanceName, temporaryPasswordFilePath, + temporaryConvertPath, temporaryConvertUuid, timeout, verboseModeEnabled); + if (!result) { + String err = String.format("The virt-v2v conversion of the instance %s failed. " + + "Please check the agent logs for the virt-v2v output", sourceInstanceName); + s_logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + String convertedBasePath = String.format("%s/%s", temporaryConvertPath, temporaryConvertUuid); + LibvirtDomainXMLParser xmlParser = parseMigratedVMXmlDomain(convertedBasePath); + + List temporaryDisks = xmlParser == null ? + getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : + getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); + + List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, + destinationStoragePools, storagePoolMgr); + + cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); + + UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, + destinationDisks, xmlParser); + return new ConvertInstanceAnswer(cmd, convertedInstanceTO); + } catch (Exception e) { + String error = String.format("Error converting instance %s from %s, due to: %s", + sourceInstanceName, sourceHypervisorType, e.getMessage()); + s_logger.error(error, e); + return new ConvertInstanceAnswer(cmd, false, error); + } finally { + s_logger.debug("Cleaning up instance conversion temporary password file"); + Script.runSimpleBashScript(String.format("rm -rf %s", temporaryPasswordFilePath)); + if (conversionTemporaryLocation instanceof NfsTO) { + s_logger.debug("Cleaning up secondary storage temporary location"); + storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); + } + } + } + + protected KVMStoragePool getTemporaryStoragePool(DataStoreTO conversionTemporaryLocation, KVMStoragePoolManager storagePoolMgr) { + if (conversionTemporaryLocation instanceof NfsTO) { + NfsTO nfsTO = (NfsTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePoolByURI(nfsTO.getUrl()); + } else { + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + } + } + + protected boolean areSourceAndDestinationHypervisorsSupported(Hypervisor.HypervisorType sourceHypervisorType, + Hypervisor.HypervisorType destinationHypervisorType) { + return destinationHypervisorType == Hypervisor.HypervisorType.KVM && + supportedInstanceConvertSourceHypervisors.contains(sourceHypervisorType); + } + + protected List getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { + List disksDefs = xmlParser.getDisks(); + disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && + x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksDefs)) { + String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + sanitizeDisksPath(disksDefs); + return getPhysicalDisksFromDefPaths(disksDefs, pool); + } + + private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { + List disks = new ArrayList<>(); + for (LibvirtVMDef.DiskDef diskDef : disksDefs) { + KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); + disks.add(physicalDisk); + } + return disks; + } + + protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { + String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); + s_logger.info(msg); + pool.refresh(); + List disksWithPrefix = pool.listPhysicalDisks() + .stream() + .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksWithPrefix)) { + msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + return disksWithPrefix; + } + + private void cleanupDisksAndDomainFromTemporaryLocation(List disks, + KVMStoragePool temporaryStoragePool, + String temporaryConvertUuid) { + for (KVMPhysicalDisk disk : disks) { + s_logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); + temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); + } + s_logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); + Script.runSimpleBashScript(String.format("rm -f %s/%s*.xml", temporaryStoragePool.getLocalPath(), temporaryConvertUuid)); + } + + protected boolean isInstanceConversionSupportedOnHost() { + int exitValue = Script.runSimpleBashScriptForExitValue(checkIfConversionIsSupportedCommand); + return exitValue == 0; + } + + protected void sanitizeDisksPath(List disks) { + for (LibvirtVMDef.DiskDef disk : disks) { + String[] diskPathParts = disk.getDiskPath().split("/"); + String relativePath = diskPathParts[diskPathParts.length - 1]; + disk.setDiskPath(relativePath); + } + } + + protected List moveTemporaryDisksToDestination(List temporaryDisks, + List destinationStoragePools, + KVMStoragePoolManager storagePoolMgr) { + List targetDisks = new ArrayList<>(); + if (temporaryDisks.size() != destinationStoragePools.size()) { + String warn = String.format("Discrepancy between the converted instance disks (%s) " + + "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); + s_logger.warn(warn); + } + for (int i = 0; i < temporaryDisks.size(); i++) { + String poolPath = destinationStoragePools.get(i); + KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); + if (destinationPool == null) { + String err = String.format("Could not find a storage pool by URI: %s", poolPath); + s_logger.error(err); + continue; + } + KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); + if (s_logger.isDebugEnabled()) { + String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + + " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); + s_logger.debug(msg); + } + + String destinationName = UUID.randomUUID().toString(); + + KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); + targetDisks.add(destinationDisk); + } + return targetDisks; + } + + private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, + List vmDisks, + LibvirtDomainXMLParser xmlParser) { + UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); + instanceTO.setName(baseName); + instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); + instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); + return instanceTO; + } + + private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { + List nics = new ArrayList<>(); + if (xmlParser != null) { + List interfaces = xmlParser.getInterfaces(); + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setNicId(interfaceDef.getBrName()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nics.add(nic); + } + } + return nics; + } + + protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { + List instanceDisks = new ArrayList<>(); + List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; + for (int i = 0; i< vmDisks.size(); i++) { + KVMPhysicalDisk physicalDisk = vmDisks.get(i); + KVMStoragePool storagePool = physicalDisk.getPool(); + UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + disk.setPosition(i); + Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); + disk.setDatastoreHost(storagePoolHostAndPath.first()); + disk.setDatastorePath(storagePoolHostAndPath.second()); + disk.setDatastoreName(storagePool.getUuid()); + disk.setDatastoreType(storagePool.getType().name()); + disk.setCapacity(physicalDisk.getVirtualSize()); + disk.setFileBaseName(physicalDisk.getName()); + if (CollectionUtils.isNotEmpty(diskDefs)) { + LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); + disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } else { + // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver + disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } + instanceDisks.add(disk); + } + return instanceDisks; + } + + protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { + String sourceHostIp = null; + String sourcePath = null; + String storagePoolMountPoint = Script.runSimpleBashScript(String.format("mount | grep %s", storagePool.getLocalPath())); + if (StringUtils.isNotEmpty(storagePoolMountPoint)) { + String[] res = storagePoolMountPoint.strip().split(" "); + res = res[0].split(":"); + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } + return new Pair<>(sourceHostIp, sourcePath); + } + + protected boolean performInstanceConversion(String convertInstanceUrl, String sourceInstanceName, + String temporaryPasswordFilePath, + String temporaryConvertFolder, + String temporaryConvertUuid, + long timeout, boolean verboseModeEnabled) { + Script script = new Script("virt-v2v", timeout, s_logger); + script.add("--root", "first"); + script.add("-ic", convertInstanceUrl); + script.add(sourceInstanceName); + script.add("--password-file", temporaryPasswordFilePath); + script.add("-o", "local"); + script.add("-os", temporaryConvertFolder); + script.add("-of", "qcow2"); + script.add("-on", temporaryConvertUuid); + if (verboseModeEnabled) { + script.add("-v"); + } + + String logPrefix = String.format("virt-v2v source: %s %s progress", convertInstanceUrl, sourceInstanceName); + OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(s_logger, logPrefix); + script.execute(outputLogger); + int exitValue = script.getExitValue(); + return exitValue == 0; + } + + private String createTemporaryPasswordFileAndRetrievePath(RemoteInstanceTO sourceInstance) { + String password = null; + if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + password = sourceInstance.getVcenterPassword(); + } + String passwordFile = String.format("/tmp/vmw-%s", UUID.randomUUID()); + String msg = String.format("Creating a temporary password file for VMware instance %s conversion on: %s", sourceInstance.getInstanceName(), passwordFile); + s_logger.debug(msg); + Script.runSimpleBashScriptForExitValueAvoidLogging(String.format("echo \"%s\" > %s", password, passwordFile)); + return passwordFile; + } + + private String getConvertInstanceUrl(RemoteInstanceTO sourceInstance) { + String url = null; + if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + url = getConvertInstanceUrlFromVmware(sourceInstance); + } + return url; + } + + private String getConvertInstanceUrlFromVmware(RemoteInstanceTO vmwareInstance) { + String vcenter = vmwareInstance.getVcenterHost(); + String datacenter = vmwareInstance.getDatacenterName(); + String username = vmwareInstance.getVcenterUsername(); + String host = vmwareInstance.getHostName(); + String cluster = vmwareInstance.getClusterName(); + + String encodedUsername = encodeUsername(username); + return String.format("vpx://%s@%s/%s/%s/%s?no_verify=1", + encodedUsername, vcenter, datacenter, cluster, host); + } + protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { + String xmlPath = String.format("%s.xml", installPath); + if (!new File(xmlPath).exists()) { + String err = String.format("Conversion failed. Unable to find the converted XML domain, expected %s", xmlPath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + InputStream is = new BufferedInputStream(new FileInputStream(xmlPath)); + String xml = IOUtils.toString(is, Charset.defaultCharset()); + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + try { + parser.parseDomainXML(xml); + return parser; + } catch (RuntimeException e) { + String err = String.format("Error parsing the converted instance XML domain at %s: %s", xmlPath, e.getMessage()); + s_logger.error(err, e); + s_logger.debug(xml); + return null; + } + } + + protected String encodeUsername(String username) { + return URLEncoder.encode(username, Charset.defaultCharset()); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java new file mode 100644 index 00000000000..14c63b54858 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -0,0 +1,310 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ConvertInstanceCommand; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtConvertInstanceCommandWrapperTest { + + @Spy + private LibvirtConvertInstanceCommandWrapper convertInstanceCommandWrapper = Mockito.spy(LibvirtConvertInstanceCommandWrapper.class); + + @Mock + private LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + private KVMStoragePool temporaryPool; + @Mock + private KVMStoragePool destinationPool; + @Mock + private PrimaryDataStoreTO primaryDataStore; + @Mock + private NfsTO secondaryDataStore; + @Mock + private KVMStoragePoolManager storagePoolManager; + + private static final String secondaryPoolUrl = "nfs://192.168.1.1/secondary"; + private static final String vmName = "VmToImport"; + private static final String hostName = "VmwareHost1"; + private static final String vmwareVcenter = "192.168.1.2"; + private static final String vmwareDatacenter = "Datacenter"; + private static final String vmwareCluster = "Cluster"; + private static final String vmwareUsername = "administrator@vsphere.local"; + private static final String vmwarePassword = "password"; + + @Before + public void setUp() { + Mockito.when(secondaryDataStore.getUrl()).thenReturn(secondaryPoolUrl); + Mockito.when(libvirtComputingResourceMock.getStoragePoolMgr()).thenReturn(storagePoolManager); + Mockito.when(storagePoolManager.getStoragePoolByURI(secondaryPoolUrl)).thenReturn(temporaryPool); + KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); + KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); + } + + @Test + public void testIsInstanceConversionSupportedOnHost() { + try (MockedStatic + + diff --git a/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java b/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java index 654f87ec5bc..03a9f548236 100644 --- a/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java +++ b/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java @@ -22,6 +22,7 @@ package com.cloud.utils.script; import java.io.BufferedReader; import java.io.IOException; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; /** @@ -138,4 +139,31 @@ public abstract class OutputInterpreter { } } + public static class LineByLineOutputLogger extends OutputInterpreter { + private Logger logger; + private String logPrefix; + + public LineByLineOutputLogger(Logger logger) { + this.logger = logger; + } + + public LineByLineOutputLogger(Logger logger, String logPrefix) { + this.logger = logger; + this.logPrefix = logPrefix; + } + + @Override + public boolean drain() { + return true; + } + + @Override + public String interpret(BufferedReader reader) throws IOException { + String line; + while ((line = reader.readLine()) != null) { + logger.info(StringUtils.isNotBlank(logPrefix) ? String.format("(%s) %s", logPrefix, line) : line); + } + return null; + } + } } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index d9e33cf1f5f..b42acb2e3f4 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -56,6 +56,7 @@ public class Script implements Callable { private volatile boolean _isTimeOut = false; private boolean _passwordCommand = false; + private boolean avoidLoggingCommand = false; private static final ScheduledExecutorService s_executors = Executors.newScheduledThreadPool(10, new NamedThreadFactory("Script")); @@ -73,6 +74,10 @@ public class Script implements Callable { return _process.exitValue(); } + public void setAvoidLoggingCommand(boolean avoid) { + avoidLoggingCommand = avoid; + } + public Script(String command, Duration timeout, Logger logger) { this(command, timeout.getMillis(), logger); } @@ -204,9 +209,10 @@ public class Script implements Callable { public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); - String commandLine = buildCommandLine(command); - _logger.debug(String.format("Executing command [%s].", commandLine.split(KeyStoreUtils.KS_FILENAME)[0])); + if (_logger.isDebugEnabled() && !avoidLoggingCommand) { + _logger.debug(String.format("Executing command [%s].", commandLine.split(KeyStoreUtils.KS_FILENAME)[0])); + } try { _logger.trace(String.format("Creating process for command [%s].", commandLine)); @@ -518,14 +524,19 @@ public class Script implements Callable { } public static int runSimpleBashScriptForExitValue(String command) { - return runSimpleBashScriptForExitValue(command, 0); + return runSimpleBashScriptForExitValue(command, 0, false); } - public static int runSimpleBashScriptForExitValue(String command, int timeout) { + public static int runSimpleBashScriptForExitValueAvoidLogging(String command) { + return runSimpleBashScriptForExitValue(command, 0, true); + } + + public static int runSimpleBashScriptForExitValue(String command, int timeout, boolean avoidLogging) { Script s = new Script("/bin/bash", timeout); s.add("-c"); s.add(command); + s.setAvoidLoggingCommand(avoidLogging); String result = s.execute(null); if (result == null || result.trim().isEmpty()) diff --git a/vmware-base/pom.xml b/vmware-base/pom.xml index 8d7b7165dfd..623d470d760 100644 --- a/vmware-base/pom.xml +++ b/vmware-base/pom.xml @@ -78,5 +78,11 @@ maven-artifact 3.6.3 + + org.apache.cloudstack + cloud-core + 4.19.0.0-SNAPSHOT + compile + diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index ade82275376..d8c7e8a61ea 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -159,14 +161,24 @@ public class DatacenterMO extends BaseMO { return null; } - public List> getAllVmsOnDatacenter() throws Exception { - List> vms = new ArrayList>(); - + public List getAllVmsOnDatacenter() throws Exception { + List vms = new ArrayList<>(); List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"}); if (ocs != null) { for (ObjectContent oc : ocs) { - String vmName = oc.getPropSet().get(0).getVal().toString(); - vms.add(new Pair(oc.getObj(), vmName)); + ManagedObjectReference vmMor = oc.getObj(); + if (vmMor != null) { + VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor); + try { + if (!vmMo.isTemplate()) { + HostMO hostMO = vmMo.getRunningHost(); + UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo); + vms.add(unmanagedInstance); + } + } catch (Exception e) { + s_logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e); + } + } } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 841d914af32..8301dd2a561 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -26,8 +26,11 @@ import java.io.StringWriter; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.GregorianCalendar; +import java.util.HashMap; import java.util.List; import java.util.Random; import java.util.UUID; @@ -37,6 +40,30 @@ import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.hypervisor.vmware.mo.ClusterMO; +import com.cloud.hypervisor.vmware.mo.DatastoreFile; +import com.cloud.hypervisor.vmware.mo.DistributedVirtualSwitchMO; +import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.net.NetUtils; +import com.vmware.vim25.DatastoreInfo; +import com.vmware.vim25.DistributedVirtualPort; +import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; +import com.vmware.vim25.GuestInfo; +import com.vmware.vim25.GuestNicInfo; +import com.vmware.vim25.HostPortGroupSpec; +import com.vmware.vim25.NasDatastoreInfo; +import com.vmware.vim25.VMwareDVSPortSetting; +import com.vmware.vim25.VirtualDeviceFileBackingInfo; +import com.vmware.vim25.VirtualIDEController; +import com.vmware.vim25.VirtualMachineGuestOsIdentifier; +import com.vmware.vim25.VirtualMachineToolsStatus; +import com.vmware.vim25.VirtualSCSIController; +import com.vmware.vim25.VmwareDistributedVirtualSwitchPvlanSpec; +import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -763,4 +790,268 @@ public class VmwareHelper { } return host; } + + public static UnmanagedInstanceTO getUnmanagedInstance(VmwareHypervisorHost hyperHost, VirtualMachineMO vmMo) { + UnmanagedInstanceTO instance = null; + try { + instance = new UnmanagedInstanceTO(); + instance.setName(vmMo.getVmName()); + instance.setInternalCSName(vmMo.getInternalCSName()); + instance.setCpuCores(vmMo.getConfigSummary().getNumCpu()); + instance.setCpuCoresPerSocket(vmMo.getCoresPerSocket()); + instance.setCpuSpeed(vmMo.getConfigSummary().getCpuReservation()); + instance.setMemory(vmMo.getConfigSummary().getMemorySizeMB()); + instance.setOperatingSystemId(vmMo.getVmGuestInfo().getGuestId()); + + ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), hyperHost.getHyperHostCluster()); + instance.setClusterName(clusterMo.getName()); + instance.setHostName(hyperHost.getHyperHostName()); + + if (StringUtils.isEmpty(instance.getOperatingSystemId())) { + instance.setOperatingSystemId(vmMo.getConfigSummary().getGuestId()); + } + VirtualMachineGuestOsIdentifier osIdentifier = VirtualMachineGuestOsIdentifier.OTHER_GUEST; + try { + osIdentifier = VirtualMachineGuestOsIdentifier.fromValue(instance.getOperatingSystemId()); + } catch (IllegalArgumentException iae) { + if (StringUtils.isNotEmpty(instance.getOperatingSystemId()) && instance.getOperatingSystemId().contains("64")) { + osIdentifier = VirtualMachineGuestOsIdentifier.OTHER_GUEST_64; + } + } + instance.setOperatingSystem(vmMo.getGuestInfo().getGuestFullName()); + if (StringUtils.isEmpty(instance.getOperatingSystem())) { + instance.setOperatingSystem(vmMo.getConfigSummary().getGuestFullName()); + } + UnmanagedInstanceTO.PowerState powerState = UnmanagedInstanceTO.PowerState.PowerUnknown; + if (vmMo.getPowerState().toString().equalsIgnoreCase("POWERED_ON")) { + powerState = UnmanagedInstanceTO.PowerState.PowerOn; + instance.setCpuSpeed(vmMo.getRuntimeInfo().getMaxCpuUsage() / instance.getCpuCores()); + } + if (vmMo.getPowerState().toString().equalsIgnoreCase("POWERED_OFF")) { + powerState = UnmanagedInstanceTO.PowerState.PowerOff; + } + instance.setPowerState(powerState); + instance.setDisks(getUnmanageInstanceDisks(vmMo)); + instance.setNics(getUnmanageInstanceNics(hyperHost, vmMo)); + } catch (Exception e) { + s_logger.info("Unable to retrieve unmanaged instance info. " + e.getMessage()); + } + return instance; + } + + protected static List getUnmanageInstanceDisks(VirtualMachineMO vmMo) { + List instanceDisks = new ArrayList<>(); + VirtualDisk[] disks = null; + try { + disks = vmMo.getAllDiskDevice(); + } catch (Exception e) { + s_logger.info("Unable to retrieve unmanaged instance disks. " + e.getMessage()); + } + if (disks != null) { + for (VirtualDevice diskDevice : disks) { + try { + if (diskDevice instanceof VirtualDisk) { + UnmanagedInstanceTO.Disk instanceDisk = new UnmanagedInstanceTO.Disk(); + VirtualDisk disk = (VirtualDisk) diskDevice; + instanceDisk.setDiskId(disk.getDiskObjectId()); + instanceDisk.setLabel(disk.getDeviceInfo() != null ? disk.getDeviceInfo().getLabel() : ""); + instanceDisk.setFileBaseName(vmMo.getVmdkFileBaseName(disk)); + instanceDisk.setImagePath(getAbsoluteVmdkFile(disk)); + instanceDisk.setCapacity(disk.getCapacityInBytes()); + instanceDisk.setPosition(diskDevice.getUnitNumber()); + DatastoreFile file = new DatastoreFile(getAbsoluteVmdkFile(disk)); + if (StringUtils.isNoneEmpty(file.getFileBaseName(), file.getDatastoreName())) { + VirtualMachineDiskInfo diskInfo = vmMo.getDiskInfoBuilder().getDiskInfoByBackingFileBaseName(file.getFileBaseName(), file.getDatastoreName()); + instanceDisk.setChainInfo(GsonHelper.getGsonLogger().toJson(diskInfo)); + } + for (VirtualDevice device : vmMo.getAllDeviceList()) { + if (diskDevice.getControllerKey() == device.getKey()) { + if (device instanceof VirtualIDEController) { + instanceDisk.setController(DiskControllerType.getType(device.getClass().getSimpleName()).toString()); + instanceDisk.setControllerUnit(((VirtualIDEController) device).getBusNumber()); + } else if (device instanceof VirtualSCSIController) { + instanceDisk.setController(DiskControllerType.getType(device.getClass().getSimpleName()).toString()); + instanceDisk.setControllerUnit(((VirtualSCSIController) device).getBusNumber()); + } else { + instanceDisk.setController(DiskControllerType.none.toString()); + } + break; + } + } + if (disk.getBacking() instanceof VirtualDeviceFileBackingInfo) { + VirtualDeviceFileBackingInfo diskBacking = (VirtualDeviceFileBackingInfo) disk.getBacking(); + ManagedObjectReference morDs = diskBacking.getDatastore(); + DatastoreInfo info = (DatastoreInfo)vmMo.getContext().getVimClient().getDynamicProperty(diskBacking.getDatastore(), "info"); + if (info instanceof NasDatastoreInfo) { + NasDatastoreInfo dsInfo = (NasDatastoreInfo) info; + instanceDisk.setDatastoreName(dsInfo.getName()); + if (dsInfo.getNas() != null) { + instanceDisk.setDatastoreHost(dsInfo.getNas().getRemoteHost()); + instanceDisk.setDatastorePath(dsInfo.getNas().getRemotePath()); + instanceDisk.setDatastoreType(dsInfo.getNas().getType()); + } + } else { + instanceDisk.setDatastoreName(info.getName()); + } + } + s_logger.info(vmMo.getName() + " " + disk.getDeviceInfo().getLabel() + " " + disk.getDeviceInfo().getSummary() + " " + disk.getDiskObjectId() + " " + disk.getCapacityInKB() + " " + instanceDisk.getController()); + instanceDisks.add(instanceDisk); + } + } catch (Exception e) { + s_logger.info("Unable to retrieve unmanaged instance disk info. " + e.getMessage()); + } + } + Collections.sort(instanceDisks, new Comparator() { + @Override + public int compare(final UnmanagedInstanceTO.Disk disk1, final UnmanagedInstanceTO.Disk disk2) { + return extractInt(disk1) - extractInt(disk2); + } + + int extractInt(UnmanagedInstanceTO.Disk disk) { + String num = disk.getLabel().replaceAll("\\D", ""); + // return 0 if no digits found + return num.isEmpty() ? 0 : Integer.parseInt(num); + } + }); + } + return instanceDisks; + } + + private static List getUnmanageInstanceNics(VmwareHypervisorHost hyperHost, VirtualMachineMO vmMo) { + List instanceNics = new ArrayList<>(); + + HashMap> guestNicMacIPAddressMap = new HashMap<>(); + try { + GuestInfo guestInfo = vmMo.getGuestInfo(); + if (guestInfo.getToolsStatus() == VirtualMachineToolsStatus.TOOLS_OK) { + for (GuestNicInfo nicInfo: guestInfo.getNet()) { + if (CollectionUtils.isNotEmpty(nicInfo.getIpAddress())) { + List ipAddresses = new ArrayList<>(); + for (String ipAddress : nicInfo.getIpAddress()) { + if (NetUtils.isValidIp4(ipAddress)) { + ipAddresses.add(ipAddress); + } + } + guestNicMacIPAddressMap.put(nicInfo.getMacAddress(), ipAddresses); + } + } + } else { + s_logger.info(String.format("Unable to retrieve guest nics for instance: %s from VMware tools as tools status: %s", vmMo.getName(), guestInfo.getToolsStatus().toString())); + } + } catch (Exception e) { + s_logger.info("Unable to retrieve guest nics for instance from VMware tools. " + e.getMessage()); + } + VirtualDevice[] nics = null; + try { + nics = vmMo.getNicDevices(); + } catch (Exception e) { + s_logger.info("Unable to retrieve unmanaged instance nics. " + e.getMessage()); + } + if (nics != null) { + for (VirtualDevice nic : nics) { + try { + VirtualEthernetCard ethCardDevice = (VirtualEthernetCard) nic; + s_logger.error(nic.getClass().getCanonicalName() + " " + nic.getBacking().getClass().getCanonicalName() + " " + ethCardDevice.getMacAddress()); + UnmanagedInstanceTO.Nic instanceNic = new UnmanagedInstanceTO.Nic(); + instanceNic.setNicId(ethCardDevice.getDeviceInfo().getLabel()); + if (ethCardDevice instanceof VirtualPCNet32) { + instanceNic.setAdapterType(VirtualEthernetCardType.PCNet32.toString()); + } else if (ethCardDevice instanceof VirtualVmxnet2) { + instanceNic.setAdapterType(VirtualEthernetCardType.Vmxnet2.toString()); + } else if (ethCardDevice instanceof VirtualVmxnet3) { + instanceNic.setAdapterType(VirtualEthernetCardType.Vmxnet3.toString()); + } else { + instanceNic.setAdapterType(VirtualEthernetCardType.E1000.toString()); + } + instanceNic.setMacAddress(ethCardDevice.getMacAddress()); + if (guestNicMacIPAddressMap.containsKey(instanceNic.getMacAddress())) { + instanceNic.setIpAddress(guestNicMacIPAddressMap.get(instanceNic.getMacAddress())); + } + if (ethCardDevice.getSlotInfo() != null) { + instanceNic.setPciSlot(ethCardDevice.getSlotInfo().toString()); + } + VirtualDeviceBackingInfo backing = ethCardDevice.getBacking(); + if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) { + VirtualEthernetCardDistributedVirtualPortBackingInfo backingInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing; + DistributedVirtualSwitchPortConnection port = backingInfo.getPort(); + String portKey = port.getPortKey(); + String portGroupKey = port.getPortgroupKey(); + String dvSwitchUuid = port.getSwitchUuid(); + + s_logger.debug("NIC " + nic.toString() + " is connected to dvSwitch " + dvSwitchUuid + " pg " + portGroupKey + " port " + portKey); + + ManagedObjectReference dvSwitchManager = vmMo.getContext().getVimClient().getServiceContent().getDvSwitchManager(); + ManagedObjectReference dvSwitch = vmMo.getContext().getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid); + + // Get all ports + DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria(); + criteria.setInside(true); + criteria.getPortgroupKey().add(portGroupKey); + List dvPorts = vmMo.getContext().getVimClient().getService().fetchDVPorts(dvSwitch, criteria); + + for (DistributedVirtualPort dvPort : dvPorts) { + // Find the port for this NIC by portkey + if (portKey.equals(dvPort.getKey())) { + VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort.getConfig().getSetting(); + if (settings.getVlan() instanceof VmwareDistributedVirtualSwitchVlanIdSpec) { + VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan(); + s_logger.trace("Found port " + dvPort.getKey() + " with vlan " + vlanId.getVlanId()); + if (vlanId.getVlanId() > 0 && vlanId.getVlanId() < 4095) { + instanceNic.setVlan(vlanId.getVlanId()); + } + } else if (settings.getVlan() instanceof VmwareDistributedVirtualSwitchPvlanSpec) { + VmwareDistributedVirtualSwitchPvlanSpec pvlanSpec = (VmwareDistributedVirtualSwitchPvlanSpec) settings.getVlan(); + s_logger.trace("Found port " + dvPort.getKey() + " with pvlan " + pvlanSpec.getPvlanId()); + if (pvlanSpec.getPvlanId() > 0 && pvlanSpec.getPvlanId() < 4095) { + DistributedVirtualSwitchMO dvSwitchMo = new DistributedVirtualSwitchMO(vmMo.getContext(), dvSwitch); + Pair vlanDetails = dvSwitchMo.retrieveVlanFromPvlan(pvlanSpec.getPvlanId(), dvSwitch); + if (vlanDetails != null && vlanDetails.first() != null && vlanDetails.second() != null) { + instanceNic.setVlan(vlanDetails.first()); + instanceNic.setPvlan(pvlanSpec.getPvlanId()); + instanceNic.setPvlanType(vlanDetails.second().toString()); + } + } + } + break; + } + } + } else if (backing instanceof VirtualEthernetCardNetworkBackingInfo) { + VirtualEthernetCardNetworkBackingInfo backingInfo = (VirtualEthernetCardNetworkBackingInfo) backing; + instanceNic.setNetwork(backingInfo.getDeviceName()); + if (hyperHost instanceof HostMO) { + HostMO hostMo = (HostMO) hyperHost; + HostPortGroupSpec portGroupSpec = hostMo.getHostPortGroupSpec(backingInfo.getDeviceName()); + instanceNic.setVlan(portGroupSpec.getVlanId()); + } + } + instanceNics.add(instanceNic); + } catch (Exception e) { + s_logger.info("Unable to retrieve unmanaged instance nic info. " + e.getMessage()); + } + } + Collections.sort(instanceNics, new Comparator() { + @Override + public int compare(final UnmanagedInstanceTO.Nic nic1, final UnmanagedInstanceTO.Nic nic2) { + return extractInt(nic1) - extractInt(nic2); + } + + int extractInt(UnmanagedInstanceTO.Nic nic) { + String num = nic.getNicId().replaceAll("\\D", ""); + // return 0 if no digits found + return num.isEmpty() ? 0 : Integer.parseInt(num); + } + }); + } + return instanceNics; + } + + public static String getAbsoluteVmdkFile(VirtualDisk disk) { + String vmdkAbsFile = null; + VirtualDeviceBackingInfo backingInfo = disk.getBacking(); + if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo) backingInfo; + vmdkAbsFile = diskBackingInfo.getFileName(); + } + return vmdkAbsFile; + } } diff --git a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareHelperTest.java b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareHelperTest.java index 1376fdd0ab9..3908f8b0be3 100644 --- a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareHelperTest.java +++ b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareHelperTest.java @@ -20,6 +20,13 @@ package com.cloud.hypervisor.vmware.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import com.vmware.vim25.DatastoreInfo; +import com.vmware.vim25.Description; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -29,11 +36,45 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; import com.vmware.vim25.VirtualDisk; +import java.util.List; + @RunWith(MockitoJUnitRunner.class) public class VmwareHelperTest { @Mock private VirtualMachineMO virtualMachineMO; + private static final String diskLabel = "disk1"; + private static final String diskFileBaseName = "xyz.vmdk"; + private static final String dataStoreName = "Datastore"; + private static final String vmName = "VM1"; + + @Before + public void setUp() throws Exception { + VirtualDiskFlatVer2BackingInfo backingInfo = Mockito.mock(VirtualDiskFlatVer2BackingInfo.class); + Mockito.when(backingInfo.getFileName()).thenReturn("abc"); + Mockito.when(backingInfo.getDatastore()).thenReturn(Mockito.mock(ManagedObjectReference.class)); + VirtualDisk disk = Mockito.mock(VirtualDisk.class); + VirtualDisk[] disks = new VirtualDisk[1]; + disks[0] = disk; + Description description = Mockito.mock(Description.class); + Mockito.when(description.getLabel()).thenReturn(diskLabel); + Mockito.when(description.getSummary()).thenReturn(""); + Mockito.when(disk.getBacking()).thenReturn(backingInfo); + Mockito.when(disk.getDeviceInfo()).thenReturn(description); + Mockito.when(virtualMachineMO.getAllDiskDevice()).thenReturn(disks); + Mockito.when(virtualMachineMO.getVmdkFileBaseName(disk)).thenReturn(diskFileBaseName); + + DatastoreInfo datastoreInfo = Mockito.mock(DatastoreInfo.class); + Mockito.when(datastoreInfo.getName()).thenReturn(dataStoreName); + VmwareClient client = Mockito.mock(VmwareClient.class); + Mockito.when(client.getDynamicProperty(Mockito.any(ManagedObjectReference.class), Mockito.anyString())) + .thenReturn(datastoreInfo); + VmwareContext context = Mockito.mock(VmwareContext.class); + Mockito.when(context.getVimClient()).thenReturn(client); + Mockito.when(virtualMachineMO.getContext()).thenReturn(context); + Mockito.when(virtualMachineMO.getName()).thenReturn(vmName); + } + @Test public void prepareDiskDeviceTestNotLimitingIOPS() throws Exception { Mockito.when(virtualMachineMO.getIDEDeviceControllerKey()).thenReturn(1); @@ -54,4 +95,14 @@ public class VmwareHelperTest { VirtualDisk virtualDisk = (VirtualDisk) VmwareHelper.prepareDiskDevice(virtualMachineMO, null, -1, new String[1], null, 0, 0, Long.valueOf(0)); assertNull(virtualDisk.getStorageIOAllocation()); } + + @Test + public void testGetUnmanageInstanceDisks() { + List disks = VmwareHelper.getUnmanageInstanceDisks(virtualMachineMO); + Assert.assertEquals(1, disks.size()); + UnmanagedInstanceTO.Disk disk = disks.get(0); + Assert.assertEquals(diskLabel, disk.getLabel()); + Assert.assertEquals(diskFileBaseName, disk.getFileBaseName()); + Assert.assertEquals(dataStoreName, disk.getDatastoreName()); + } }