diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index d70eeb87653..6785c365329 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -74,7 +74,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch); List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 267cef2169a..08b82cbb45b 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -24,6 +24,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -621,11 +622,19 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } @Override - public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch) { List templates = listAllReadySystemVMTemplates(zoneId); if (CollectionUtils.isEmpty(templates)) { return null; } + if (StringUtils.isNotBlank(preferredArch)) { + // Sort the templates by preferred architecture first + templates = templates.stream() + .sorted(Comparator.comparing( + x -> !x.getArch().getType().equalsIgnoreCase(preferredArch) + )) + .collect(Collectors.toList()); + } if (hypervisorType == HypervisorType.Any) { return templates.get(0); } diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index 7f151730c9c..3c8e4c046ae 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -316,4 +317,24 @@ public class VMTemplateDaoImplTest { verify(searchCriteria).setParameters("extensionId", extensionId); verify(templateDao).customSearchIncludingRemoved(eq(searchCriteria), isNull()); } + + @Test + public void testFindSystemVMReadyTemplate() { + Long zoneId = 1L; + VMTemplateVO systemVmTemplate1 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate1.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate2 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate2.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate3 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate3.getArch()).thenReturn(CPU.CPUArch.arm64); + Mockito.when(systemVmTemplate3.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + List templates = Arrays.asList(systemVmTemplate1, systemVmTemplate2, systemVmTemplate3); + Mockito.when(hostDao.listDistinctHypervisorTypes(zoneId)).thenReturn(Arrays.asList(Hypervisor.HypervisorType.KVM)); + SearchBuilder sb = mock(SearchBuilder.class); + templateDao.readySystemTemplateSearch = sb; + when(sb.create()).thenReturn(mock(SearchCriteria.class)); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()); + Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch()); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index d95bc0b9784..dc092608ff3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -482,8 +482,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return null; } - public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType, Map templateNodeTypeMap, KubernetesClusterNodeType nodeType) { - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); + public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType, Map templateNodeTypeMap, KubernetesClusterNodeType nodeType, + KubernetesSupportedVersion clusterKubernetesVersion) { + String systemVMPreferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenter.getId()); + VMTemplateVO cksIso = clusterKubernetesVersion != null ? + templateDao.findById(clusterKubernetesVersion.getIsoId()) : + null; + String preferredArchitecture = getCksClusterPreferredArch(systemVMPreferredArchitecture, cksIso); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType, preferredArchitecture); if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) { logger.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter)); template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId())); @@ -510,6 +516,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + datacenterId); } + protected String getCksClusterPreferredArch(String systemVMPreferredArchitecture, VMTemplateVO cksIso) { + if (cksIso == null) { + return systemVMPreferredArchitecture; + } + String cksIsoArchName = cksIso.getArch().name(); + return cksIsoArchName.equals(systemVMPreferredArchitecture) ? systemVMPreferredArchitecture : cksIsoArchName; + } + protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) { List rules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO); for (FirewallRuleVO rule : rules) { @@ -1541,11 +1555,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } Map templateNodeTypeMap = cmd.getTemplateNodeTypeMap(); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, DEFAULT); - final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL); - final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER); - final VMTemplateVO etcdNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, ETCD); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, DEFAULT, clusterKubernetesVersion); + final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL, clusterKubernetesVersion); + final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER, clusterKubernetesVersion); + final VMTemplateVO etcdNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, ETCD, clusterKubernetesVersion); final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId(), asNumber); + final SecurityGroup finalSecurityGroup = securityGroup; final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 1f2f42c3fe6..9be5f4c0776 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -36,6 +36,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -257,7 +258,8 @@ public class KubernetesClusterActionWorker { DataCenterVO dataCenterVO = dataCenterDao.findById(zoneId); VMTemplateVO template = templateDao.findById(templateId); Hypervisor.HypervisorType type = template.getHypervisorType(); - this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, null, KubernetesClusterNodeType.DEFAULT); + KubernetesSupportedVersionVO kubernetesSupportedVersion = kubernetesSupportedVersionDao.findById(this.kubernetesCluster.getKubernetesVersionId()); + this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, null, KubernetesClusterNodeType.DEFAULT, kubernetesSupportedVersion); this.controlNodeTemplate = templateDao.findById(this.kubernetesCluster.getControlNodeTemplateId()); this.workerNodeTemplate = templateDao.findById(this.kubernetesCluster.getWorkerNodeTemplateId()); this.etcdTemplate = templateDao.findById(this.kubernetesCluster.getEtcdNodeTemplateId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 9bad38c2af3..d392612547b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -212,7 +212,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); HostVO h = hp.first(); - if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType())) { + if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType()) || !h.getArch().equals(clusterTemplate.getArch())) { continue; } hostDao.loadHostTags(h); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 57715cc9309..0793482b8f1 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -21,6 +21,7 @@ package com.cloud.kubernetes.cluster; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenter; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -422,4 +423,22 @@ public class KubernetesClusterManagerImplTest { Assert.assertEquals(workerOffering, mapping.get(WORKER.name())); Assert.assertEquals(controlOffering, mapping.get(CONTROL.name())); } + + @Test + public void testGetCksClusterPreferredArchDifferentArchsPreferCKSIsoArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.arm64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.arm64.name(), cksClusterPreferredArch); + } + + @Test + public void testGetCksClusterPreferredArchSameArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.amd64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.amd64.name(), cksClusterPreferredArch); + } } diff --git a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java index c1d4153fbd1..ac8d6a58f0c 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java +++ b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java @@ -175,10 +175,11 @@ public class StorageVmSharedFSLifeCycle implements SharedFSLifeCycle { customParameterMap.put("maxIopsDo", maxIops.toString()); } List keypairs = new ArrayList(); + String preferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId); for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final Hypervisor.HypervisorType hypervisor = iter.next(); - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor, preferredArchitecture); if (template == null && !iter.hasNext()) { throw new CloudRuntimeException(String.format("Unable to find the systemvm template for %s or it was not downloaded in %s.", hypervisor.toString(), zone.toString())); } diff --git a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java index 5f30dab383f..c64e8c05c99 100644 --- a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java +++ b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java @@ -235,7 +235,7 @@ public class StorageVmSharedFSLifeCycleTest { when(serviceOfferingDao.findById(s_serviceOfferingId)).thenReturn(serviceOffering); VMTemplateVO template = mock(VMTemplateVO.class); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(template); + when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM, ResourceManager.SystemVmPreferredArchitecture.defaultValue())).thenReturn(template); when(template.getId()).thenReturn(s_templateId); return sharedFS; @@ -302,7 +302,6 @@ public class StorageVmSharedFSLifeCycleTest { when(dataCenterDao.findById(s_zoneId)).thenReturn(zone); when(resourceMgr.getSupportedHypervisorTypes(s_zoneId, false, null)).thenReturn(List.of(Hypervisor.HypervisorType.KVM)); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(null); lifeCycle.deploySharedFS(sharedFS, s_networkId, s_diskOfferingId, s_size, s_minIops, s_maxIops); }