From c8d6e505393868b18ea8f3fd7dd75732c292998d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 17 Aug 2023 10:42:42 +0200 Subject: [PATCH] VMware: add support for 8.0b (8.0.0.2), 8.0c (8.0.0.3) (#7380) * VMware: add support for 8.0b (8.0.0.2) * VMware 8: add new guest os mappings in VirtualMachineGuestOsIdentifier The full list can be found at https://developer.vmware.com/apis/1355/vsphere * VMware: get guest os mappings of parent version * VMware8: remove guest os mappings for 8.0.0.2 * VMware8: fix code smells * vmware: remove annotations in VmwareVmImplementerTest which caused 0.0% code coverage * VMware8: add a unit test case * VMware: add support for 8.0c (8.0.0.3) * VMware8: move to CloudStackVersion.getVMwareParentVersion * VMware: add support for 8.0u1 (8.0.1.0) * Copy engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java from PR 6979 * Copy engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java from PR 6979 * VMware: ignore the last number in VMware versions * VMware: copy guest os mapping from 8.0 to 8.0.1 * VMware: add unit tests in VmwareVmImplementerTest.java * Copy engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java from PR 6979 * VMware8: retry vm poweron if fails due to exception "File system specific implementation of Ioctl[file] failed" This fixes a weird issue on vmware8. When power on a vm, sometimes it fails due to error 2023-04-27 07:04:43,207 ERROR [c.c.h.v.r.VmwareResource] (DirectAgent-442:ctx-cdd42b03 10.0.32.133, job-105/job-106, cmd: StartCommand) (logid:8a24a607) StartCommand failed due to [Exception: java.lang.RuntimeException Message: File system specific implementation of Ioctl[file] failed ]. java.lang.RuntimeException: File system specific implementation of Ioctl[file] failed at com.cloud.hypervisor.vmware.util.VmwareClient.waitForTask(VmwareClient.java:426) at com.cloud.hypervisor.vmware.mo.VirtualMachineMO.powerOn(VirtualMachineMO.java:288) in vmware.log on ESXi host, it shows 2023-04-27T09:20:41.713Z In(05)+ vmx - Power on failure messages: File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - Failed to lock the file 2023-04-27T09:20:41.713Z In(05)+ vmx - Cannot open the disk '/vmfs/volumes/7b29c876-ac102328/i-2-167-VM/ROOT-167.vmdk' or one of the snapshot disks it depends on. 2023-04-27T09:20:41.713Z In(05)+ vmx - Module 'Disk' power on failed. 2023-04-27T09:20:41.713Z In(05)+ vmx - Failed to start the virtual machine. There is a KB article for it, but I still do not know why and how to fix it. https://kb.vmware.com/s/article/1004232 * VMware: extract to method powerOnVM * vmware: fix mistake in logs * vmware8: use curl instead of wget to fix test failures Traceback (most recent call last): File "/root/test_internal_lb.py", line 555, in test_01_internallb_roundrobin_1VPC_3VM_HTTP_port80 self.execute_internallb_roundrobin_tests(vpc_offering) File "/root/test_internal_lb.py", line 641, in execute_internallb_roundrobin_tests client_vm, applb.sourceipaddress, max_http_requests) File "/root/test_internal_lb.py", line 497, in run_ssh_test_accross_hosts (e, clienthost.public_ip)) AssertionError: list index out of range: SSH failed for VM with IP Address: 10.0.52.187 and sshClient: DEBUG: {Cmd: /usr/bin/wget -T3 -qO- --user=admin --password=password http://10.1.2.253:8081/admin?stats via Host: 10.0.52.188} {returns: ["/usr/bin/wget: '/usr/lib/libpcre.so.1' is not an ELF file", "/usr/bin/wget: can't load library 'libpcre.so.1'"]} * VMware: correct guest OS names in hypervisor mappings for VMware 8.0 el9 and variants were introduced by https://github.com/apache/cloudstack/pull/7059 they are supported with guest os identifiers since VMware 8.0 see https://vdc-repo.vmware.com/vmwb-repository/dcr-public/c476b64b-c93c-4b21-9d76-be14da0148f9/04ca12ad-59b9-4e1c-8232-fd3d4276e52c/SDK/vsphere-ws/docs/ReferenceGuide/vim.vm.GuestOsDescriptor.GuestOsIdentifier.html * VMware: add Ubuntu 20.04 and 22.04 support for vmware 7.0+ * PR7380: only add guest os mappings for Ubuntu 20.04 * PR7380: Correct RHEL9 guest os names and others for VMware 8.0 * PR7380: correct guest os names on 8.0.0.1 as well * PR7380: remove Windows 12 and Windows Server 2025 which are not released yet --- .../dao/HypervisorCapabilitiesDaoImpl.java | 7 ++ .../storage/dao/GuestOSHypervisorDao.java | 2 + .../storage/dao/GuestOSHypervisorDaoImpl.java | 13 ++ .../java/com/cloud/upgrade/GuestOsMapper.java | 51 +++++++- .../upgrade/dao/Upgrade41800to41810.java | 114 ++++++++++++++++++ .../META-INF/db/schema-41800to41810.sql | 3 + .../com/cloud/upgrade/GuestOsMapperTest.java | 84 +++++++++++++ .../hypervisor/guru/VmwareVmImplementer.java | 16 ++- .../vmware/resource/VmwareResource.java | 21 +++- .../guru/VmwareVmImplementerTest.java | 52 +++++++- test/integration/smoke/test_internal_lb.py | 7 +- .../cloudstack/utils/CloudStackVersion.java | 17 +++ .../utils/CloudStackVersionTest.java | 20 +++ 13 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java diff --git a/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java b/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java index 9a1f29014f9..e8272825213 100644 --- a/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.dao; import java.util.ArrayList; import java.util.List; +import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,6 +74,12 @@ public class HypervisorCapabilitiesDaoImpl extends GenericDaoBase sc = HypervisorTypeAndVersionSearch.create(); sc.setParameters("hypervisorType", hypervisorType); sc.setParameters("hypervisorVersion", hypervisorVersion); + HypervisorCapabilitiesVO result = findOneBy(sc); + if (result != null || !HypervisorType.VMware.equals(hypervisorType) || + CloudStackVersion.getVMwareParentVersion(hypervisorVersion) == null) { + return result; + } + sc.setParameters("hypervisorVersion", CloudStackVersion.getVMwareParentVersion(hypervisorVersion)); return findOneBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java index 17c6b3c959d..47a71433a32 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java @@ -40,4 +40,6 @@ public interface GuestOSHypervisorDao extends GenericDao listHypervisorSupportedVersionsFromMinimumVersion(String hypervisorType, String hypervisorVersion); + + List listByHypervisorTypeAndVersion(String hypervisorType, String hypervisorVersion); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java index ae3ae9aceb8..65f17c29fbc 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java @@ -39,6 +39,7 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase userDefinedMappingSearch; protected final SearchBuilder guestOsNameSearch; protected final SearchBuilder availableHypervisorVersionSearch; + protected final SearchBuilder hypervisorTypeAndVersionSearch; public GuestOSHypervisorDaoImpl() { guestOsSearch = createSearchBuilder(); @@ -73,6 +74,11 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase listByHypervisorTypeAndVersion(String hypervisorType, String hypervisorVersion) { + SearchCriteria sc = hypervisorTypeAndVersionSearch.create(); + sc.setParameters("hypervisor_type", hypervisorType); + sc.setParameters("hypervisor_version", hypervisorVersion); + return listIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java index def77c58f82..739eb32261c 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java @@ -17,6 +17,7 @@ package com.cloud.upgrade; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import java.sql.Connection; @@ -26,6 +27,7 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorMapping; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; @@ -94,7 +96,7 @@ public class GuestOsMapper { } } - private boolean addGuestOs(long categoryId, String displayName) { + public boolean addGuestOs(long categoryId, String displayName) { LOG.debug("Adding guest OS with category id: " + categoryId + " and display name: " + displayName); GuestOSVO guestOS = new GuestOSVO(); guestOS.setCategoryId(categoryId); @@ -116,7 +118,7 @@ public class GuestOsMapper { return; } - LOG.debug("Adding guest OS hypervisor mapping - " + mapping.toString()); + LOG.debug("Adding guest OS hypervisor mapping - " + mapping.toString() + ", for guest OS with id - " + guestOsId); GuestOSHypervisorVO guestOsMapping = new GuestOSHypervisorVO(); guestOsMapping.setHypervisorType(mapping.getHypervisorType()); guestOsMapping.setHypervisorVersion(mapping.getHypervisorVersion()); @@ -198,4 +200,49 @@ public class GuestOsMapper { LOG.warn("Invalid Guest OS hypervisor mapping"); return false; } + + /** + * Copies guest OS mappings from src version to dest version for the hypervisor (use this to copy all mappings from older version to newer version during upgrade) + * @return true if copied successfully, else false. + */ + public boolean copyGuestOSHypervisorMappings(HypervisorType hypervisorType, String srcVersion, String destVersion) { + if (hypervisorType == HypervisorType.None || hypervisorType == HypervisorType.Any) { + LOG.warn("Unable to copy, invalid hypervisor"); + return false; + } + + if (StringUtils.isBlank(srcVersion) || StringUtils.isBlank(destVersion)) { + LOG.warn("Unable to copy, invalid hypervisor version details"); + return false; + } + + List guestOSHypervisorMappingsForSrcVersion = guestOSHypervisorDao.listByHypervisorTypeAndVersion(hypervisorType.toString(), srcVersion); + if (CollectionUtils.isEmpty(guestOSHypervisorMappingsForSrcVersion)) { + LOG.warn(String.format("Unable to copy, couldn't find guest OS mappings for hypervisor: %s and src version: %s", hypervisorType.toString(), srcVersion)); + return false; + } + + LOG.debug(String.format("Adding guest OS mappings for hypervisor: %s and version: %s, from version: %s ", hypervisorType.toString(), destVersion, srcVersion)); + for (GuestOSHypervisorVO guestOSHypervisorMapping : guestOSHypervisorMappingsForSrcVersion) { + GuestOSHypervisorMapping mapping = new GuestOSHypervisorMapping(hypervisorType.toString(), destVersion, guestOSHypervisorMapping.getGuestOsName()); + addGuestOsHypervisorMapping(mapping, guestOSHypervisorMapping.getGuestOsId()); + } + return true; + } + + public void updateGuestOsNameInHypervisorMapping(long categoryId, String displayName, GuestOSHypervisorMapping mapping) { + if (!isValidGuestOSHypervisorMapping(mapping)) { + return; + } + + long guestOsId = getGuestOsId(categoryId, displayName); + if (guestOsId == 0) { + LOG.error(String.format("no guest os found for category %d and name %s, skipping mapping it to %s/%s", guestOsId, displayName, mapping.getHypervisorType(), mapping.getHypervisorVersion())); + return; + } + + GuestOSHypervisorVO guestOsMapping = guestOSHypervisorDao.findByOsIdAndHypervisor(guestOsId, mapping.getHypervisorType(), mapping.getHypervisorVersion()); + guestOsMapping.setGuestOsName(mapping.getGuestOsName()); + guestOSHypervisorDao.update(guestOsMapping.getId(), guestOsMapping); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index ab493da8cd8..a2733215060 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.upgrade.dao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.GuestOSHypervisorMapping; +import com.cloud.upgrade.GuestOsMapper; import com.cloud.upgrade.SystemVmTemplateRegistration; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -58,6 +61,8 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate @Override public void performDataMigration(Connection conn) { fixForeignKeyNames(conn); + updateGuestOsMappings(conn); + copyGuestOsMappingsToVMware80u1(); } @Override @@ -86,6 +91,115 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate } } + private void updateGuestOsMappings(Connection conn) { + LOG.debug("Updating guest OS mappings"); + + GuestOsMapper guestOsMapper = new GuestOsMapper(); + List mappings = new ArrayList<>(); + + LOG.debug("Adding Ubuntu 20.04 support for VMware 6.5+"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.5", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.1", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.2", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.3", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.1.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.2.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.3.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "8.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + + LOG.debug("Adding Ubuntu 22.04 support for KVM and VMware 6.5+"); + mappings.add(new GuestOSHypervisorMapping("KVM", "default", "Ubuntu 22.04 LTS")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.5", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.1", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.2", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.3", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.1.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.2.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.3.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "8.0", "ubuntu64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(10, "Ubuntu 22.04 LTS", mappings); + mappings.clear(); + + LOG.debug("Correcting guest OS names in hypervisor mappings for VMware 8.0 ad 8.0.0.1"); + final String hypervisorVMware = Hypervisor.HypervisorType.VMware.name(); + final String hypervisorVersionVmware8 = "8.0"; + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "AlmaLinux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "almalinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Oracle Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "oracleLinux9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Rocky Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "rockylinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "AlmaLinux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "almalinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Oracle Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "oracleLinux9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Rocky Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "rockylinux_64Guest")); + + LOG.debug("Correcting guest OS names in hypervisor mappings for Red Hat Enterprise Linux 9"); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.1.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.2.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.3.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "rhel9_64Guest")); + + LOG.debug("Adding new guest OS ids in hypervisor mappings for VMware 8.0"); + // Add support for darwin22_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "darwin22_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "macOS 13 (64-bit)", mappings); + mappings.clear(); + + // Add support for darwin23_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "darwin23_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "macOS 14 (64-bit)", mappings); + mappings.clear(); + + // Add support for debian12_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "debian12_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(2, "Debian GNU/Linux 12 (64-bit)", mappings); + mappings.clear(); + + // Add support for debian12Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "debian12Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(2, "Debian GNU/Linux 12 (32-bit)", mappings); + mappings.clear(); + + // Add support for freebsd14_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "freebsd14_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(9, "FreeBSD 14 (64-bit)", mappings); + mappings.clear(); + + // Add support for freebsd14Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "freebsd14Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(9, "FreeBSD 14 (32-bit)", mappings); + mappings.clear(); + + // Add support for other6xLinux64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "other6xLinux64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "Linux 6.x Kernel (64-bit)", mappings); + mappings.clear(); + + // Add support for other6xLinuxGuest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "other6xLinuxGuest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "Linux 6.x Kernel (32-bit)", mappings); + mappings.clear(); + + // Add support for vmkernel8Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "vmkernel8Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "VMware ESXi 8.0", mappings); + mappings.clear(); + + // Add support for windows11_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "windows11_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(6, "Windows 11 (64-bit)", mappings); + mappings.clear(); + } + + private void copyGuestOsMappingsToVMware80u1() { + LOG.debug("Copying guest OS mappings from VMware 8.0 to VMware 8.0.1"); + GuestOsMapper guestOsMapper = new GuestOsMapper(); + guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.VMware, "8.0", "8.0.1"); + } + private void fixForeignKeyNames(Connection conn) { //Alter foreign key name for user_vm table from fk_user_data_id to fk_user_vm__user_data_id (if exists) List keys = new ArrayList(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql index 07ac5c8f166..e07e12871f3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql @@ -19,6 +19,9 @@ -- Schema upgrade from 4.18.0.0 to 4.18.1.0 --; +-- Add support for VMware 8.0u1 (8.0.1.x) +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.1', 1024, 0, 59, 64, 1, 1); + -- Update conserve_mode of the default network offering for Tungsten Fabric (this fixes issue #7241) UPDATE `cloud`.`network_offerings` SET conserve_mode = 0 WHERE unique_name ='DefaultTungstenFarbicNetworkOffering'; diff --git a/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java b/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java new file mode 100644 index 00000000000..fa6c693fe63 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.GuestOSHypervisorVO; +import com.cloud.storage.dao.GuestOSHypervisorDao; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(PowerMockRunner.class) +public class GuestOsMapperTest { + + @Spy + @InjectMocks + GuestOsMapper guestOsMapper = new GuestOsMapper(); + + @Mock + GuestOSHypervisorDao guestOSHypervisorDao; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testCopyGuestOSHypervisorMappingsFailures() { + boolean result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.Any, "6.0", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.None, "6.0", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", ""); + Assert.assertFalse(result); + + Mockito.when(guestOSHypervisorDao.listByHypervisorTypeAndVersion(Mockito.anyString(), Mockito.anyString())).thenReturn(null); + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", "7.0"); + Assert.assertFalse(result); + } + + @Test + public void testCopyGuestOSHypervisorMappingsSuccess() { + GuestOSHypervisorVO guestOSHypervisorVO = Mockito.mock(GuestOSHypervisorVO.class); + List guestOSHypervisorVOS = new ArrayList<>(); + guestOSHypervisorVOS.add(guestOSHypervisorVO); + Mockito.when(guestOSHypervisorDao.listByHypervisorTypeAndVersion(Mockito.anyString(), Mockito.anyString())).thenReturn(guestOSHypervisorVOS); + Mockito.when(guestOSHypervisorVO.getGuestOsName()).thenReturn("centos"); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOSHypervisorDao.persist(guestOsMapping)).thenReturn(guestOsMapping); + + boolean result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", "7.0"); + Assert.assertTrue(result); + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 100e3d416a7..990a1875a57 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -47,6 +47,7 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; +import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang.BooleanUtils; import org.apache.log4j.Logger; @@ -165,7 +166,7 @@ class VmwareVmImplementer { GuestOSHypervisorVO guestOsMapping = null; if (host != null) { - guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), host.getHypervisorVersion()); + guestOsMapping = getGuestOsMapping(guestOS, host.getHypervisorVersion()); } if (guestOsMapping == null || host == null) { to.setPlatformEmulator(null); @@ -405,4 +406,17 @@ class VmwareVmImplementer { return listForSort.toArray(new NicTO[0]); } + + protected GuestOSHypervisorVO getGuestOsMapping(GuestOSVO guestOS , String hypervisorVersion) { + GuestOSHypervisorVO guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), hypervisorVersion); + if (guestOsMapping == null) { + LOGGER.debug(String.format("Cannot find guest os mappings for guest os \"%s\" on VMware %s", guestOS.getDisplayName(), hypervisorVersion)); + String parentVersion = CloudStackVersion.getVMwareParentVersion(hypervisorVersion); + if (parentVersion != null) { + guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), parentVersion); + LOGGER.debug(String.format("Found guest os mappings for guest os \"%s\" on VMware %s: %s", guestOS.getDisplayName(), parentVersion, guestOsMapping)); + } + } + return guestOsMapping; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 8b833b8221a..01d6f6816e0 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2627,7 +2627,9 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes // // Power-on VM // - if (!vmMo.powerOn()) { + if (powerOnVM(vmMo, vmInternalCSName, vmNameOnVcenter)) { + s_logger.debug(String.format("VM %s has been started successfully with hostname %s.", vmInternalCSName, vmNameOnVcenter)); + } else { throw new Exception("Failed to start VM. vmName: " + vmInternalCSName + " with hostname " + vmNameOnVcenter); } @@ -2699,6 +2701,23 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } } + private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception { + int retry = 20; + while (retry-- > 0) { + try { + return vmMo.powerOn(); + } catch (Exception e) { + s_logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e); + if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) { + s_logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter)); + } else { + throw e; + } + } + } + return false; + } + private boolean multipleIsosAtached(DiskTO[] sortedDisks) { return Arrays.stream(sortedDisks).filter(disk -> disk.getType() == Volume.Type.ISO).count() > 1; } diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java index 309eb6d35c4..f1647808c94 100755 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.verify; import java.util.HashMap; import java.util.Map; -import org.apache.cloudstack.framework.config.ConfigKey; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,17 +32,15 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.support.AnnotationConfigContextLoader; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.storage.GuestOSHypervisorVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.dao.GuestOSHypervisorDao; import com.cloud.vm.VmDetailConstants; @RunWith(PowerMockRunner.class) -@PrepareForTest({ConfigKey.class, VmwareVmImplementer.class}) -@ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class VmwareVmImplementerTest { @Spy @@ -52,6 +50,9 @@ public class VmwareVmImplementerTest { @Mock VirtualMachineTO vmTO; + @Mock + GuestOSHypervisorDao guestOsHypervisorDao; + private Map vmDetails = new HashMap(); @Before @@ -145,4 +146,43 @@ public class VmwareVmImplementerTest { executeAndVerifyTest(false, false, "false", false); } + @Test + public void testGetGuestOsMapping1() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(guestOsMapping); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertEquals(guestOsMapping, result); + } + + @Test + public void testGetGuestOsMapping2() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(null); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1")).thenReturn(guestOsMapping); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertEquals(guestOsMapping, result); + } + + @Test + public void testGetGuestOsMapping3() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(null); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1")).thenReturn(null); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertNull(result); + } + + @Test + public void testGetGuestOsMapping4() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0")).thenReturn(null); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0"); + Assert.assertNull(result); + } } diff --git a/test/integration/smoke/test_internal_lb.py b/test/integration/smoke/test_internal_lb.py index 5864f7321bb..8dc341f4d14 100644 --- a/test/integration/smoke/test_internal_lb.py +++ b/test/integration/smoke/test_internal_lb.py @@ -485,7 +485,7 @@ class TestInternalLb(cloudstackTestCase): try: for x in range(0, max_requests): - cmd_test_http = "/usr/bin/wget -T2 -qO- http://" + \ + cmd_test_http = "curl --connect-timeout 3 -L http://" + \ lb_address + "/ 2>/dev/null" # self.debug( "SSH into VM public address: %s and port: %s" # %(.public_ip, vm.public_port)) @@ -677,9 +677,8 @@ class TestInternalLb(cloudstackTestCase): url = "http://" + stats_ip + ":" + \ settings["stats_port"] + settings["stats_uri"] - get_contents = "/usr/bin/wget -T3 -qO- --user=" + \ - settings["username"] + " --password=" + \ - settings["password"] + " " + url + get_contents = "curl --connect-timeout 3 -L --user %s:%s %s" \ + % (settings["username"], settings["password"], url) try: self.logger.debug( "Trying to connect to the haproxy stats url %s" % url) diff --git a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java index 91c5a9758ea..e29bd9c4e17 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java @@ -272,4 +272,21 @@ public final class CloudStackVersion implements Comparable { public String toString() { return Joiner.on(".").join(asList()); } + + /** + * Get the parent version of VMware hypervisor version + * @since 4.18.1.0 + */ + public static String getVMwareParentVersion(String hypervisorVersion) { + try { + CloudStackVersion version = CloudStackVersion.parse(hypervisorVersion); + String parentVersion = String.format("%s.%s", version.getMajorRelease(), version.getMinorRelease()); + if (version.getPatchRelease() != 0) { + parentVersion = String.format("%s.%s", parentVersion, version.getPatchRelease()); + } + return parentVersion; + } catch (Exception ex) { + return null; + } + } } diff --git a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java index eb7a76a2c0f..dabaf9bc97d 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java @@ -21,6 +21,7 @@ package org.apache.cloudstack.utils; import com.google.common.testing.EqualsTester; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -204,4 +205,23 @@ public final class CloudStackVersionTest { assertEquals(expected, CloudStackVersion.trimRouterVersion(value)); } + + private void verifyGetVMwareParentVersion(String hypervisorVersion, String expectedParentVersion) { + if (expectedParentVersion == null) { + Assert.assertNull(CloudStackVersion.getVMwareParentVersion(hypervisorVersion)); + } else { + Assert.assertEquals(CloudStackVersion.getVMwareParentVersion(hypervisorVersion), expectedParentVersion); + } + } + @Test + public void testGetParentVersion() { + verifyGetVMwareParentVersion(null, null); + verifyGetVMwareParentVersion("6.5", null); + verifyGetVMwareParentVersion("6.7.3", "6.7.3"); + verifyGetVMwareParentVersion("7.0.3.0", "7.0.3"); + verifyGetVMwareParentVersion("8.0", null); + verifyGetVMwareParentVersion("8.0.0", "8.0"); + verifyGetVMwareParentVersion("8.0.0.2", "8.0"); + verifyGetVMwareParentVersion("8.0.1.0", "8.0.1"); + } }