From 98c51a6d3d65b63ba9165090261d6d90035acce9 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 29 Sep 2020 09:15:29 +0200 Subject: [PATCH 1/2] server: check guest os preference of last host when start a vm (#4338) If vm has last host_id specified, cloudstack will try to start vm on it at first. However, host tag is checked, but guest os preference is not checked. for new vm, it will be deployed to the preferred host as we expect. Fixes: #3554 (comment) --- .../deploy/DeploymentPlanningManagerImpl.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 26b36157dcd..b1e081057e2 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -89,10 +89,12 @@ import com.cloud.exception.AffinityConflictException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.gpu.GPU; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; @@ -102,6 +104,7 @@ import com.cloud.resource.ResourceState; import com.cloud.service.ServiceOfferingDetailsVO; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.GuestOSVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; @@ -165,6 +168,8 @@ StateListener { private long _hostReservationReleasePeriod = 60L * 60L * 1000L; // one hour by default @Inject protected VMReservationDao _reservationDao; + @Inject + HostDetailsDao _hostDetailsDao; private static final long INITIAL_RESERVATION_RELEASE_CHECKER_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds protected long _nodeId = -1; @@ -411,14 +416,7 @@ StateListener { } } else { if (host.getStatus() == Status.Up && host.getResourceState() == ResourceState.Enabled) { - boolean hostTagsMatch = true; - if(offering.getHostTag() != null){ - _hostDao.loadHostTags(host); - if (!(host.getHostTags() != null && host.getHostTags().contains(offering.getHostTag()))) { - hostTagsMatch = false; - } - } - if (hostTagsMatch) { + if (checkVmProfileAndHost(vmProfile, host)) { long cluster_id = host.getClusterId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); @@ -489,8 +487,6 @@ StateListener { } else { s_logger.debug("The last host of this VM does not have enough capacity"); } - } else { - s_logger.debug("Service Offering host tag does not match the last host of this VM"); } } else { s_logger.debug("The last host of this VM is not UP or is not enabled, host status is: " + host.getStatus().name() + ", host resource state is: " + @@ -569,6 +565,31 @@ StateListener { return null; } + private boolean checkVmProfileAndHost(final VirtualMachineProfile vmProfile, final HostVO host) { + ServiceOffering offering = vmProfile.getServiceOffering(); + if (offering.getHostTag() != null) { + _hostDao.loadHostTags(host); + if (!(host.getHostTags() != null && host.getHostTags().contains(offering.getHostTag()))) { + s_logger.debug("Service Offering host tag does not match the last host of this VM"); + return false; + } + } + long guestOSId = vmProfile.getTemplate().getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + if (guestOS != null) { + long guestOSCategoryId = guestOS.getCategoryId(); + DetailVO hostDetail = _hostDetailsDao.findDetail(host.getId(), "guest.os.category.id"); + if (hostDetail != null) { + String guestOSCategoryIdString = hostDetail.getValue(); + if (String.valueOf(guestOSCategoryId) != guestOSCategoryIdString) { + s_logger.debug("The last host has different guest.os.category.id than guest os category of VM, skipping"); + return false; + } + } + } + return true; + } + @Override public void checkForNonDedicatedResources(VirtualMachineProfile vmProfile, DataCenter dc, ExcludeList avoids) { boolean isExplicit = false; From 9391fa9b6b54adec7dc02802e1706648bc950cbc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 29 Sep 2020 14:31:41 +0530 Subject: [PATCH 2/2] cks: assorted fixes, test refactoring (#4269) Fixes #4265 Fixes for: template selection network cleanup capacity check specific patch version during deployment Signed-off-by: Abhishek Kumar --- .../cloud/storage/dao/VMTemplateDaoImpl.java | 8 +- .../cluster/KubernetesClusterManagerImpl.java | 18 +- .../KubernetesClusterDestroyWorker.java | 33 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../KubernetesClusterStartWorker.java | 12 +- .../version/KubernetesVersionManagerImpl.java | 8 +- .../AddKubernetesSupportedVersionCmd.java | 2 +- .../DeleteKubernetesSupportedVersionCmd.java | 9 +- .../cluster/CreateKubernetesClusterCmd.java | 2 +- .../cluster/DeleteKubernetesClusterCmd.java | 8 +- .../cluster/ScaleKubernetesClusterCmd.java | 8 +- .../cluster/StartKubernetesClusterCmd.java | 8 +- .../cluster/StopKubernetesClusterCmd.java | 8 +- .../cluster/UpgradeKubernetesClusterCmd.java | 8 +- .../util/create-kubernetes-binaries-iso.sh | 2 +- .../smoke/test_kubernetes_clusters.py | 725 ++++++++++-------- .../test_kubernetes_supported_versions.py | 16 +- tools/marvin/marvin/config/test_data.py | 290 ++++--- 18 files changed, 675 insertions(+), 492 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 1621baef2df..6caef20b2d7 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -95,6 +95,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder AccountIdSearch; protected SearchBuilder NameSearch; + protected SearchBuilder ValidNameSearch; protected SearchBuilder TmpltsInZoneSearch; protected SearchBuilder ActiveTmpltSearch; private SearchBuilder PublicSearch; @@ -138,8 +139,9 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem @Override public VMTemplateVO findValidByTemplateName(String templateName) { - SearchCriteria sc = NameSearch.create(); + SearchCriteria sc = ValidNameSearch.create(); sc.setParameters("name", templateName); + sc.setParameters("state", VirtualMachineTemplate.State.Active); return findOneBy(sc); } @@ -319,6 +321,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem UniqueNameSearch.and("uniqueName", UniqueNameSearch.entity().getUniqueName(), SearchCriteria.Op.EQ); NameSearch = createSearchBuilder(); NameSearch.and("name", NameSearch.entity().getName(), SearchCriteria.Op.EQ); + ValidNameSearch = createSearchBuilder(); + ValidNameSearch.and("name", ValidNameSearch.entity().getName(), SearchCriteria.Op.EQ); + ValidNameSearch.and("state", ValidNameSearch.entity().getState(), SearchCriteria.Op.EQ); + ValidNameSearch.and("removed", ValidNameSearch.entity().getRemoved(), SearchCriteria.Op.NULL); NameAccountIdSearch = createSearchBuilder(); NameAccountIdSearch.and("name", NameAccountIdSearch.entity().getName(), SearchCriteria.Op.EQ); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 204f2d7b83d..a6538c946c3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -292,7 +292,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", templateKey)); return false; } - final VMTemplateVO template = templateDao.findByTemplateName(templateName); + final VMTemplateVO template = templateDao.findValidByTemplateName(templateName); if (template == null) { LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster nodes", templateName)); return false; @@ -375,22 +375,22 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } private VMTemplateVO getKubernetesServiceTemplate(Hypervisor.HypervisorType hypervisorType) { - String tempalteName = null; + String templateName = null; switch (hypervisorType) { case Hyperv: - tempalteName = KubernetesClusterHyperVTemplateName.value(); + templateName = KubernetesClusterHyperVTemplateName.value(); break; case KVM: - tempalteName = KubernetesClusterKVMTemplateName.value(); + templateName = KubernetesClusterKVMTemplateName.value(); break; case VMware: - tempalteName = KubernetesClusterVMwareTemplateName.value(); + templateName = KubernetesClusterVMwareTemplateName.value(); break; case XenServer: - tempalteName = KubernetesClusterXenserverTemplateName.value(); + templateName = KubernetesClusterXenserverTemplateName.value(); break; } - return templateDao.findValidByTemplateName(tempalteName); + return templateDao.findValidByTemplateName(templateName); } private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { @@ -514,7 +514,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } boolean suitable_host_found = false; Cluster planCluster = null; - for (int i = 1; i <= nodesCount + 1; i++) { + for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); @@ -991,7 +991,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne try { deployDestination = plan(totalNodeCount, zone, serviceOffering); } catch (InsufficientCapacityException e) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d cluster nodes in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); } if (deployDestination == null || deployDestination.getCluster() == null) { logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone ID: %s", zone.getUuid())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index c1a095fedbe..41de39fddc2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -198,24 +198,25 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod } if (cleanupNetwork) { // if network has additional VM, cannot proceed with cluster destroy NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - logAndThrow(Level.ERROR, String.format("Failed to find network for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - List networkVMs = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(network.getId(), VirtualMachine.Type.User); - if (networkVMs.size() > clusterVMs.size()) { - logAndThrow(Level.ERROR, String.format("Network ID: %s for Kubernetes cluster ID: %s has instances using it which are not part of the Kubernetes cluster", network.getUuid(), kubernetesCluster.getUuid())); - } - for (VMInstanceVO vm : networkVMs) { - boolean vmFoundInKubernetesCluster = false; - for (KubernetesClusterVmMap clusterVM : clusterVMs) { - if (vm.getId() == clusterVM.getVmId()) { - vmFoundInKubernetesCluster = true; - break; + if (network != null) { + List networkVMs = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(network.getId(), VirtualMachine.Type.User); + if (networkVMs.size() > clusterVMs.size()) { + logAndThrow(Level.ERROR, String.format("Network ID: %s for Kubernetes cluster ID: %s has instances using it which are not part of the Kubernetes cluster", network.getUuid(), kubernetesCluster.getUuid())); + } + for (VMInstanceVO vm : networkVMs) { + boolean vmFoundInKubernetesCluster = false; + for (KubernetesClusterVmMap clusterVM : clusterVMs) { + if (vm.getId() == clusterVM.getVmId()) { + vmFoundInKubernetesCluster = true; + break; + } + } + if (!vmFoundInKubernetesCluster) { + logAndThrow(Level.ERROR, String.format("VM ID: %s which is not a part of Kubernetes cluster ID: %s is using Kubernetes cluster network ID: %s", vm.getUuid(), kubernetesCluster.getUuid(), network.getUuid())); } } - if (!vmFoundInKubernetesCluster) { - logAndThrow(Level.ERROR, String.format("VM ID: %s which is not a part of Kubernetes cluster ID: %s is using Kubernetes cluster network ID: %s", vm.getUuid(), kubernetesCluster.getUuid(), network.getUuid())); - } + } else { + LOGGER.error(String.format("Failed to find network for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } if (LOGGER.isInfoEnabled()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 8b6df146bff..831bebe5ff2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -205,7 +205,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; - for (int i = 1; i <= nodesCount + 1; i++) { + for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 1ef5d0c788c..90c03751279 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -76,10 +76,19 @@ import com.google.common.base.Strings; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { + private KubernetesSupportedVersion kubernetesClusterVersion; + public KubernetesClusterStartWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); } + public KubernetesSupportedVersion getKubernetesClusterVersion() { + if (kubernetesClusterVersion == null) { + kubernetesClusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + } + return kubernetesClusterVersion; + } + private Pair> getKubernetesMasterIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException { String masterIp = null; Map requestedIps = null; @@ -105,7 +114,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private boolean isKubernetesVersionSupportsHA() { boolean haSupported = false; - final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + KubernetesSupportedVersion version = getKubernetesClusterVersion(); if (version != null) { try { if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { @@ -161,6 +170,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); + initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); k8sMasterConfig = k8sMasterConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); return k8sMasterConfig; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 5f2e42b11d5..72a1c379487 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -181,10 +181,10 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); } if(!isSemanticVersion(v1)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); + throw new IllegalArgumentException(String.format("Invalid version format, %s. Semantic version should be specified in MAJOR.MINOR.PATCH format", v1)); } if(!isSemanticVersion(v2)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); + throw new IllegalArgumentException(String.format("Invalid version format, %s. Semantic version should be specified in MAJOR.MINOR.PATCH format", v2)); } String[] thisParts = v1.split("\\."); String[] thatParts = v2.split("\\."); @@ -287,10 +287,10 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne final Integer minimumCpu = cmd.getMinimumCpu(); final Integer minimumRamSize = cmd.getMinimumRamSize(); if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { - throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_CPU_NUMBER)); + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %d vCPUs required.", ApiConstants.MIN_CPU_NUMBER, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU)); } if (minimumRamSize == null || minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_MEMORY)); + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %dMB memory required", ApiConstants.MIN_MEMORY, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index a85e6ee064a..ece9d5aae5d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -61,7 +61,7 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm private String name; @Parameter(name = ApiConstants.SEMANTIC_VERSION, type = CommandType.STRING, required = true, - description = "the semantic version of the Kubernetes version") + description = "the semantic version of the Kubernetes version. It needs to be specified in MAJOR.MINOR.PATCH format") private String semanticVersion; @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index 02489147c65..8e4ca7242d7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -83,7 +83,14 @@ public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd implements @Override public String getEventDescription() { - return "Deleting Kubernetes supported version " + getId(); + String description = "Deleting Kubernetes supported version"; + KubernetesSupportedVersion version = _entityMgr.findById(KubernetesSupportedVersion.class, getId()); + if (version != null) { + description += String.format(" ID: %s", version.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 32b07c4c36a..54e307c0c5b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -259,7 +259,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Override public String getEventDescription() { - return "creating Kubernetes cluster. Cluster Id: " + getEntityId(); + return "Creating Kubernetes cluster. Cluster Id: " + getEntityId(); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index 4f32138758e..d6bc75c9c3d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -102,8 +102,14 @@ public class DeleteKubernetesClusterCmd extends BaseAsyncCmd { @Override public String getEventDescription() { + String description = "Deleting Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Deleting Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 90ccfa41917..994b3997022 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -94,8 +94,14 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { @Override public String getEventDescription() { + String description = "Scaling Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Scaling Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index 1ce2fe09c10..d61a942e7f9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -73,8 +73,14 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd { @Override public String getEventDescription() { + String description = "Starting Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Starting Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index ba2649f863e..d4f49bd225f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -74,8 +74,14 @@ public class StopKubernetesClusterCmd extends BaseAsyncCmd { @Override public String getEventDescription() { + String description = "Stopping Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Stopping Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index 2c99b005ff4..4a2e0b1c9a8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -84,8 +84,14 @@ public class UpgradeKubernetesClusterCmd extends BaseAsyncCmd { @Override public String getEventDescription() { + String description = "Upgrading Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Upgrading Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index bf97f0662f7..d7d9c16310d 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -86,7 +86,7 @@ if [ $? -ne 0 ]; then fi fi mkdir -p "${working_dir}/docker" -output=`${k8s_dir}/kubeadm config images list` +output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` while read -r line; do echo "Downloading docker image $line ---" sudo docker pull "$line" diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 492c9702a9d..2b7638d5324 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -19,29 +19,40 @@ #Import Local Modules from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.cloudstackAPI import (listInfrastructure, + listTemplates, listKubernetesSupportedVersions, addKubernetesSupportedVersion, deleteKubernetesSupportedVersion, + listKubernetesClusters, createKubernetesCluster, stopKubernetesCluster, + startKubernetesCluster, deleteKubernetesCluster, upgradeKubernetesCluster, - scaleKubernetesCluster) + scaleKubernetesCluster, + destroyVirtualMachine, + deleteNetwork) from marvin.cloudstackException import CloudstackAPIException -from marvin.codes import FAILED +from marvin.codes import PASS, FAILED from marvin.lib.base import (Template, ServiceOffering, + Account, Configurations) from marvin.lib.utils import (cleanup_resources, + validateList, random_gen) -from marvin.lib.common import (get_zone) +from marvin.lib.common import (get_zone, + get_domain) from marvin.sshClient import SshClient from nose.plugins.attrib import attr +from marvin.lib.decoratorGenerators import skipTestIf import time _multiprocess_shared_ = True +k8s_cluster = None + class TestKubernetesCluster(cloudstackTestCase): @classmethod @@ -54,111 +65,89 @@ class TestKubernetesCluster(cloudstackTestCase): cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.cks_template_name_key = "cloud.kubernetes.cluster.template.name." + cls.hypervisor.lower() + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + cls.hypervisorNotSupported = True cls.setup_failed = False - - cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, - name="cloud.kubernetes.service.enabled")[0].value - if cls.initial_configuration_cks_enabled not in ["true", True]: - cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") - Configurations.update(cls.apiclient, - "cloud.kubernetes.service.enabled", - "true") - cls.restartServer() - - cls.cks_template = None - cls.initial_configuration_cks_template_name = None - cls.cks_service_offering = None - - cls.kubernetes_version_ids = [] - if cls.setup_failed == False: - try: - cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://download.cloudstack.org/cks/setup-1.14.9.iso') - cls.kubernetes_version_ids.append(cls.kubernetes_version_1.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.14.9.iso, %s" % e) - if cls.setup_failed == False: - try: - cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://download.cloudstack.org/cks/setup-1.15.0.iso') - cls.kubernetes_version_ids.append(cls.kubernetes_version_2.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.15.0.iso, %s" % e) - if cls.setup_failed == False: - try: - cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion('1.16.0', 'http://download.cloudstack.org/cks/setup-1.16.0.iso') - cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.16.0.iso, %s" % e) - if cls.setup_failed == False: - try: - cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion('1.16.3', 'http://download.cloudstack.org/cks/setup-1.16.3.iso') - cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.16.3.iso, %s" % e) - - cks_template_data = { - "name": "Kubernetes-Service-Template", - "displaytext": "Kubernetes-Service-Template", - "format": "qcow2", - "hypervisor": "kvm", - "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", - "ispublic": "True", - "isextractable": "True" - } - cks_template_data_details = [] - if cls.hypervisor.lower() == "vmware": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" - cks_template_data["format"] = "OVA" - cks_template_data_details = [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] - elif cls.hypervisor.lower() == "xenserver": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" - cks_template_data["format"] = "VHD" - elif cls.hypervisor.lower() == "kvm": - cks_template_data["requireshvm"] = "True" - if cls.setup_failed == False: - cls.cks_template = Template.register( - cls.apiclient, - cks_template_data, - zoneid=cls.zone.id, - hypervisor=cls.hypervisor, - details=cks_template_data_details - ) - cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) - try: - cls.waitForTemplateReadyState(cls.cks_template.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get CKS template in ready state, {}, {}".format(cks_template_data["url"], e)) - - cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, - name=cls.cks_template_name_key)[0].value - Configurations.update(cls.apiclient, - cls.cks_template_name_key, - cls.cks_template.name) - - cks_offering_data = { - "name": "CKS-Instance", - "displaytext": "CKS Instance", - "cpunumber": 2, - "cpuspeed": 1000, - "memory": 2048, - } - cks_offering_data["name"] = cks_offering_data["name"] + '-' + random_gen() - if cls.setup_failed == False: - cls.cks_service_offering = ServiceOffering.create( - cls.apiclient, - cks_offering_data - ) - cls._cleanup = [] - if cls.cks_template != None: - cls._cleanup.append(cls.cks_template) - if cls.cks_service_offering != None: - cls._cleanup.append(cls.cks_service_offering) + cls.kubernetes_version_ids = [] + + if cls.hypervisorNotSupported == False: + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, + name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "true") + cls.restartServer() + + cls.cks_template = None + cls.initial_configuration_cks_template_name = None + cls.cks_service_offering = None + + if cls.setup_failed == False: + try: + cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.14.9"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_1.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.14.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.14.9"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.15.0"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_2.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) + + if cls.setup_failed == False: + cls.cks_template = cls.getKubernetesTemplate() + if cls.cks_template == FAILED: + assert False, "getKubernetesTemplate() failed to return template for hypervisor %s" % cls.hypervisor + cls.setup_failed = True + else: + cls._cleanup.append(cls.cks_template) + + if cls.setup_failed == False: + cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, + name=cls.cks_template_name_key)[0].value + Configurations.update(cls.apiclient, + cls.cks_template_name_key, + cls.cks_template.name) + + cks_offering_data = cls.services["cks_service_offering"] + cks_offering_data["name"] = 'CKS-Instance-' + random_gen() + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cls._cleanup.append(cls.cks_service_offering) + cls.domain = get_domain(cls.apiclient) + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) return @classmethod @@ -173,14 +162,12 @@ class TestKubernetesCluster(cloudstackTestCase): cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: # Restore original CKS template - if cls.initial_configuration_cks_template_name != None: + if cls.cks_template != None: + cls.cks_template.delete(cls.apiclient) + if cls.hypervisorNotSupported == False and cls.initial_configuration_cks_template_name != None: Configurations.update(cls.apiclient, cls.cks_template_name_key, cls.initial_configuration_cks_template_name) - # Delete created CKS template - if cls.setup_failed == False and cls.cks_template != None: - cls.cks_template.delete(cls.apiclient, - cls.zone.id) # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Restoring Kubernetes Service enabled value") @@ -192,7 +179,7 @@ class TestKubernetesCluster(cloudstackTestCase): cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - if version_delete_failed == True: + if version_delete_failed == True: raise Exception("Warning: Exception during cleanup, unable to delete Kubernetes supported versions") return @@ -231,28 +218,39 @@ class TestKubernetesCluster(cloudstackTestCase): return False @classmethod - def waitForTemplateReadyState(cls, template_id, retries=30, interval=60): - """Check if template download will finish""" - while retries > 0: - time.sleep(interval) - template_response = Template.list( - cls.apiclient, - id=template_id, - zoneid=cls.zone.id, - templatefilter='self' - ) + def getKubernetesTemplate(cls, cks_templates=None): - if isinstance(template_response, list): - template = template_response[0] - if not hasattr(template, 'status') or not template or not template.status: - retries = retries - 1 - continue - if 'Failed' == template.status: - raise Exception("Failed to download template: status - %s" % template.status) - elif template.status == 'Download Complete' and template.isready: - return - retries = retries - 1 - raise Exception("Template download timed out") + if cks_templates is None: + cks_templates = cls.services["cks_templates"] + + hypervisor = cls.hypervisor.lower() + + if hypervisor not in cks_templates.keys(): + cls.debug("Provided hypervisor has no CKS template") + return FAILED + + cks_template = cks_templates[hypervisor] + + cmd = listTemplates.listTemplatesCmd() + cmd.name = cks_template['name'] + cmd.templatefilter = 'all' + cmd.zoneid = cls.zone.id + cmd.hypervisor = hypervisor + templates = cls.apiclient.listTemplates(cmd) + + if validateList(templates)[0] != PASS: + details = None + if hypervisor in ["vmware"] and "details" in cks_template: + details = cks_template["details"] + template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) + template.download(cls.apiclient) + return template + + for template in templates: + if template.isready and template.ispublic: + return Template(template.__dict__) + + return FAILED @classmethod def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=30, interval=60): @@ -278,14 +276,14 @@ class TestKubernetesCluster(cloudstackTestCase): versionResponse = cls.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) return versionResponse[0] - @classmethod - def addKubernetesSupportedVersion(cls, semantic_version, iso_url): + @classmethod + def addKubernetesSupportedVersion(cls, version_service): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() - addKubernetesSupportedVersionCmd.semanticversion = semantic_version - addKubernetesSupportedVersionCmd.name = 'v' + semantic_version + '-' + random_gen() - addKubernetesSupportedVersionCmd.url = iso_url - addKubernetesSupportedVersionCmd.mincpunumber = 2 - addKubernetesSupportedVersionCmd.minmemory = 2048 + addKubernetesSupportedVersionCmd.semanticversion = version_service["semanticversion"] + addKubernetesSupportedVersionCmd.name = 'v' + version_service["semanticversion"] + '-' + random_gen() + addKubernetesSupportedVersionCmd.url = version_service["url"] + addKubernetesSupportedVersionCmd.mincpunumber = version_service["mincpunumber"] + addKubernetesSupportedVersionCmd.minmemory = version_service["minmemory"] kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id) @@ -308,14 +306,13 @@ class TestKubernetesCluster(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created templates cleanup_resources(self.apiclient, self.cleanup) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_01_deploy_kubernetes_cluster(self): """Test to deploy a new Kubernetes cluster @@ -324,75 +321,156 @@ class TestKubernetesCluster(cloudstackTestCase): # 2. The Cloud Database contains the valid information # 3. stopKubernetesCluster should stop the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now stopping it" % k8s_cluster.id) - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) + self.stopAndVerifyKubernetesCluster(k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now stopping it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully stopped, now starting it again" % k8s_cluster.id) - self.stopAndVerifyKubernetesCluster(cluster_response.id) + try: + k8s_cluster = self.startKubernetesCluster(k8s_cluster.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to start Kubernetes cluster due to: %s" % e) - self.debug("Kubernetes cluster with ID: %s successfully stopped, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.verifyKubernetesClusterState(k8s_cluster, 'Running') return @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_02_deploy_kubernetes_ha_cluster(self): - """Test to deploy a new Kubernetes cluster + @skipTestIf("hypervisorNotSupported") + def test_02_invalid_upgrade_kubernetes_cluster(self): + """Test to check for failure while tying to upgrade a Kubernetes cluster to a lower version # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information + # 1. upgradeKubernetesCluster should fail """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_3.id, 1, 2) + self.debug("Upgrading Kubernetes cluster with ID: %s to a lower version" % k8s_cluster.id) - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_3.id, 1, 2) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kubernetes_version_1.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") + except Exception as e: + self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) return @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_03_deploy_invalid_kubernetes_ha_cluster(self): - """Test to deploy a new Kubernetes cluster + @skipTestIf("hypervisorNotSupported") + def test_03_deploy_and_upgrade_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and upgrade it to newer version # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information + # 1. upgradeKubernetesCluster should return valid info for the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() + time.sleep(self.services["sleep"]) + + self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) + + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_3.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_3.id) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_04_deploy_and_scale_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it + + # Validate the following: + # 1. scaleKubernetesCluster should return valid info for the cluster when it is scaled up + # 2. scaleKubernetesCluster should return valid info for the cluster when it is scaled down + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() + + self.debug("Upscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) + + try: + k8s_cluster = self.scaleKubernetesCluster(k8s_cluster.id, 2) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upscale Kubernetes cluster due to: %s" % e) + + self.verifyKubernetesClusterScale(k8s_cluster, 2) + + self.debug("Kubernetes cluster with ID: %s successfully upscaled, now downscaling it" % k8s_cluster.id) + + try: + k8s_cluster = self.scaleKubernetesCluster(k8s_cluster.id, 1) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to downscale Kubernetes cluster due to: %s" % e) + + self.verifyKubernetesClusterScale(k8s_cluster) + + self.debug("Kubernetes cluster with ID: %s successfully downscaled" % k8s_cluster.id) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_05_delete_kubernetes_cluster(self): + """Test to delete an existing Kubernetes cluster + + # Validate the following: + # 1. deleteKubernetesCluster should delete an existing Kubernetes cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() + + self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id) + + self.deleteKubernetesClusterAndVerify(k8s_cluster.id) + + self.debug("Kubernetes cluster with ID: %s successfully deleted" % k8s_cluster.id) + + k8s_cluster = None + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_06_deploy_invalid_kubernetes_ha_cluster(self): + """Test to deploy an invalid HA Kubernetes cluster + + # Validate the following: + # 1. createKubernetesCluster should fail as version doesn't support HA + """ + if self.setup_failed == True: + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) try: cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id, 1, 2) - self.debug("Invslid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) - self.deleteKubernetesCluster(cluster_response.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id, False, True) self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") except CloudstackAPIException as e: self.debug("HA Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) @@ -400,174 +478,79 @@ class TestKubernetesCluster(cloudstackTestCase): return @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_04_deploy_and_upgrade_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and upgrade it to newer version + @skipTestIf("hypervisorNotSupported") + def test_07_deploy_kubernetes_ha_cluster(self): + """Test to deploy a new Kubernetes cluster # Validate the following: # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should return valid info for the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) - - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) - - try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_3.id) - except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(cluster_response, self.kubernetes_version_3.id) - - self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("HA Kubernetes cluster with ID: %s successfully deployed" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): + @skipTestIf("hypervisorNotSupported") + def test_08_deploy_and_upgrade_kubernetes_ha_cluster(self): """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should return valid info for the cluster + # 1. upgradeKubernetesCluster should return valid info for the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_3.id, 1, 2) - - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_3.id, 1, 2) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) + time.sleep(self.services["sleep"]) + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_4.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_4.id) except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(cluster_response, self.kubernetes_version_4.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_4.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + @skipTestIf("hypervisorNotSupported") + def test_09_delete_kubernetes_ha_cluster(self): + """Test to delete a HA Kubernetes cluster # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should fail + # 1. deleteKubernetesCluster should delete an existing HA Kubernetes cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) + self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id) - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now scaling it" % cluster_response.id) - - try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_1.id) - self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kubernetes_version_1.id) - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") - except Exception as e: - self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) - - self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_07_deploy_and_scale_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it - - # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. scaleKubernetesCluster should return valid info for the cluster when it is scaled up - # 4. scaleKubernetesCluster should return valid info for the cluster when it is scaled down - """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) - if self.setup_failed == True: - self.skipTest("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) - - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upscaling it" % cluster_response.id) - - try: - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 2) - except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to upscale Kubernetes cluster due to: %s" % e) - - self.verifyKubernetesClusterScale(cluster_response, 2) - - self.debug("Kubernetes cluster with ID: %s successfully upscaled, now downscaling it" % cluster_response.id) - - try: - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) - except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to downscale Kubernetes cluster due to: %s" % e) - - self.verifyKubernetesClusterScale(cluster_response) - - self.debug("Kubernetes cluster with ID: %s successfully downscaled, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) - - return - - def listKubernetesCluster(self, cluster_id): + def listKubernetesCluster(self, cluster_id = None): listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() - listKubernetesClustersCmd.id = cluster_id + if cluster_id != None: + listKubernetesClustersCmd.id = cluster_id clusterResponse = self.apiclient.listKubernetesClusters(listKubernetesClustersCmd) - return clusterResponse[0] + if cluster_id != None and clusterResponse != None: + return clusterResponse[0] + return clusterResponse def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd() @@ -579,6 +562,8 @@ class TestKubernetesCluster(cloudstackTestCase): createKubernetesClusterCmd.serviceofferingid = self.cks_service_offering.id createKubernetesClusterCmd.zoneid = self.zone.id createKubernetesClusterCmd.noderootdisksize = 10 + createKubernetesClusterCmd.account = self.account.name + createKubernetesClusterCmd.domainid = self.domain.id clusterResponse = self.apiclient.createKubernetesCluster(createKubernetesClusterCmd) if not clusterResponse: self.cleanup.append(clusterResponse) @@ -590,6 +575,12 @@ class TestKubernetesCluster(cloudstackTestCase): response = self.apiclient.stopKubernetesCluster(stopKubernetesClusterCmd) return response + def startKubernetesCluster(self, cluster_id): + startKubernetesClusterCmd = startKubernetesCluster.startKubernetesClusterCmd() + startKubernetesClusterCmd.id = cluster_id + response = self.apiclient.startKubernetesCluster(startKubernetesClusterCmd) + return response + def deleteKubernetesCluster(self, cluster_id): deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd() deleteKubernetesClusterCmd.id = cluster_id @@ -610,18 +601,58 @@ class TestKubernetesCluster(cloudstackTestCase): response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) return response - def verifyKubernetesCluster(self, cluster_response, name, version_id, size=1, master_nodes=1): + def getValidKubernetesCluster(self, size=1, master_nodes=1): + cluster = k8s_cluster + version = self.kubernetes_version_2 + if master_nodes != 1: + version = self.kubernetes_version_3 + valid = True + if cluster == None: + valid = False + self.debug("No existing cluster available, k8s_cluster: %s" % cluster) + if valid == True and cluster.id == None: + valid = False + self.debug("ID for existing cluster not found, k8s_cluster ID: %s" % cluster.id) + if valid == True: + cluster_id = cluster.id + cluster = self.listKubernetesCluster(cluster_id) + if cluster == None: + valid = False + self.debug("Existing cluster, k8s_cluster ID: %s not returned by list API" % cluster_id) + if valid == True: + try: + self.verifyKubernetesCluster(cluster, cluster.name, None, size, master_nodes) + self.debug("Existing Kubernetes cluster available with name %s" % cluster.name) + except AssertionError as error: + valid = False + self.debug("Existing cluster failed verification due to %s, need to deploy a new one" % error) + if valid == False: + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + try: + self.deleteAllLeftoverClusters() + cluster = self.createKubernetesCluster(name, version.id, size, master_nodes) + self.verifyKubernetesCluster(cluster, name, version.id, size, master_nodes) + except Exception as ex: + self.fail("Kubernetes cluster deployment failed: %s" % ex) + except AssertionError as err: + self.fail("Kubernetes cluster deployment failed during cluster verification: %s" % err) + return cluster + + def verifyKubernetesCluster(self, cluster_response, name, version_id=None, size=1, master_nodes=1): """Check if Kubernetes cluster is valid""" self.verifyKubernetesClusterState(cluster_response, 'Running') - self.assertEqual( - cluster_response.name, - name, - "Check KubernetesCluster name {}, {}".format(cluster_response.name, name) - ) + if name != None: + self.assertEqual( + cluster_response.name, + name, + "Check KubernetesCluster name {}, {}".format(cluster_response.name, name) + ) - self.verifyKubernetesClusterVersion(cluster_response, version_id) + if version_id != None: + self.verifyKubernetesClusterVersion(cluster_response, version_id) self.assertEqual( cluster_response.zoneid, @@ -703,21 +734,51 @@ class TestKubernetesCluster(cloudstackTestCase): "KubernetesCluster not stopped in DB, {}".format(db_cluster_state) ) - def deleteAndVerifyKubernetesCluster(self, cluster_id): + def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = False): """Delete Kubernetes cluster and check if it is really deleted""" - delete_response = self.deleteKubernetesCluster(cluster_id) + forceDeleted = False + try: + delete_response = self.deleteKubernetesCluster(cluster_id) + except Exception as e: + if forced: + cluster = self.listKubernetesCluster(cluster_id) + if cluster != None: + if cluster.state in ['Starting', 'Running', 'Upgrading', 'Scaling']: + self.stopKubernetesCluster(cluster_id) + self.deleteKubernetesCluster(cluster_id) + else: + forceDeleted = True + for cluster_vm_id in cluster.virtualmachineids: + cmd = destroyVirtualMachine.destroyVirtualMachineCmd() + cmd.id = cluster_vm_id + cmd.expunge = True + self.apiclient.destroyVirtualMachine(cmd) + cmd = deleteNetwork.deleteNetworkCmd() + cmd.id = cluster.networkid + cmd.forced = True + self.apiclient.deleteNetwork(cmd) + self.dbclient.execute("update kubernetes_cluster set state='Destroyed', removed=now() where uuid = '%s';" % cluster.id) + else: + raise Exception("Error: Exception during delete cluster : %s" % e) - self.assertEqual( - delete_response.success, - True, - "Check KubernetesCluster delete response {}, {}".format(delete_response.success, True) - ) + if verify == True and forceDeleted == False: + self.assertEqual( + delete_response.success, + True, + "Check KubernetesCluster delete response {}, {}".format(delete_response.success, True) + ) - db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] + db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] - self.assertNotEqual( - db_cluster_removed, - None, - "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) - ) + self.assertNotEqual( + db_cluster_removed, + None, + "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) + ) + + def deleteAllLeftoverClusters(self): + clusters = self.listKubernetesCluster() + if clusters != None: + for cluster in clusters: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index b2202053ebf..ffeb00599d4 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -130,12 +130,12 @@ class TestKubernetesSupportedVersion(cloudstackTestCase): # 2. The Cloud Database contains the valid information when listKubernetesSupportedVersions is called """ - version = '1.16.3' - name = 'v' + version + '-' + random_gen() + version = self.services["cks_kubernetes_versions"]["1.16.3"] + name = 'v' + version["semanticversion"] + '-' + random_gen() self.debug("Adding Kubernetes supported version with name: %s" % name) - version_response = self.addKubernetesSupportedVersion(version, name, self.zone.id, self.kubernetes_version_iso_url) + version_response = self.addKubernetesSupportedVersion(version["semanticversion"], name, self.zone.id, version["url"], version["mincpunumber"], version["minmemory"]) list_versions_response = self.listKubernetesSupportedVersion(version_response.id) @@ -147,8 +147,8 @@ class TestKubernetesSupportedVersion(cloudstackTestCase): self.assertEqual( list_versions_response.semanticversion, - version, - "Check KubernetesSupportedVersion version {}, {}".format(list_versions_response.semanticversion, version) + version["semanticversion"], + "Check KubernetesSupportedVersion version {}, {}".format(list_versions_response.semanticversion, version["semanticversion"]) ) self.assertEqual( list_versions_response.zoneid, @@ -228,14 +228,14 @@ class TestKubernetesSupportedVersion(cloudstackTestCase): self.debug("Unsupported version error check successful, API failure: %s" % e) return - def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl): + def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl, mincpunumber=2, minmemory=2048): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = version addKubernetesSupportedVersionCmd.name = name addKubernetesSupportedVersionCmd.zoneid = zoneId addKubernetesSupportedVersionCmd.url = isoUrl - addKubernetesSupportedVersionCmd.mincpunumber = 2 - addKubernetesSupportedVersionCmd.minmemory = 2048 + addKubernetesSupportedVersionCmd.mincpunumber = mincpunumber + addKubernetesSupportedVersionCmd.minmemory = minmemory versionResponse = self.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) if not versionResponse: self.cleanup.append(versionResponse) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 929741b7b0d..47087176be2 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1847,119 +1847,187 @@ test_data = { }, }, "configurableData": - { - "portableIpRange": { - "gateway": "10.223.59.1", - "netmask": "255.255.255.0", - "startip": "10.223.59.200", - "endip": "10.223.59.240", - "vlan": "1000" - }, - "netscaler": { - "ipaddress": "", - "username": "", - "password": "", - "networkdevicetype": "", - "publicinterface": "", - "privateinterface": "", - "numretries": "", - "lbdevicededicated": "False", - "lbdevicecapacity": 2, - "port": 22 - }, - "iscsi": { - "url": "", - "name": "Primary iSCSI" - }, - "host": { - "publicport": 22, - "username": "root", - "password": "password", - }, - "ldap_account": { - "email": "", - "firstname": "", - "lastname": "", - "username": "", - "password": "", - }, - "link_ldap_details": { - "domain_name": "", - "accounttype": "", - "name": "", - "type": "", - "admin": "", - "linkLdapUsername": "", - "linkLdapPassword": "", - "linkLdapNestedUser": "", - "linkLdapNestedPassword": "" + { + "portableIpRange": { + "gateway": "10.223.59.1", + "netmask": "255.255.255.0", + "startip": "10.223.59.200", + "endip": "10.223.59.240", + "vlan": "1000" + }, + "netscaler": { + "ipaddress": "", + "username": "", + "password": "", + "networkdevicetype": "", + "publicinterface": "", + "privateinterface": "", + "numretries": "", + "lbdevicededicated": "False", + "lbdevicecapacity": 2, + "port": 22 + }, + "iscsi": { + "url": "", + "name": "Primary iSCSI" + }, + "host": { + "publicport": 22, + "username": "root", + "password": "password", + }, + "ldap_account": { + "email": "", + "firstname": "", + "lastname": "", + "username": "", + "password": "", + }, + "link_ldap_details": { + "domain_name": "", + "accounttype": "", + "name": "", + "type": "", + "admin": "", + "linkLdapUsername": "", + "linkLdapPassword": "", + "linkLdapNestedUser": "", + "linkLdapNestedPassword": "" - }, - "ldap_configuration": { - "basedn": "", - "emailAttribute": "", - "userObject": "", - "usernameAttribute": "", - "hostname": "", - "port": "", - "ldapUsername": "", - "ldapPassword": "" - }, - "systemVmDelay": 120, - "setUsageConfigurationThroughTestCase": True, - "vmware_cluster": { - "hypervisor": 'VMware', - "clustertype": 'ExternalManaged', - "username": '', - "password": '', - "url": '', - "clustername": 'VMWare Cluster with Space in DC name', - }, - "upload_volume": { - "diskname": "UploadVol", - "format": "VHD", - "url": "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2", - "checksum": "", - }, - "bootableIso": - { - "displaytext": "Test Bootable ISO", - "name": "testISO", - "bootable": True, - "ispublic": False, - "url": "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso", - "ostype": 'Other Linux (64-bit)', - "mode": 'HTTP_DOWNLOAD' - }, - "setHostConfigurationForIngressRule": False, - "restartManagementServerThroughTestCase": False, - "vmxnet3template": { - "displaytext": "VMXNET3 Template", - "name": "VMXNET3 template", - "ostype": "CentOS 5.6 (64-bit)", - "isfeatured": True, + }, + "ldap_configuration": { + "basedn": "", + "emailAttribute": "", + "userObject": "", + "usernameAttribute": "", + "hostname": "", + "port": "", + "ldapUsername": "", + "ldapPassword": "" + }, + "systemVmDelay": 120, + "setUsageConfigurationThroughTestCase": True, + "vmware_cluster": { + "hypervisor": 'VMware', + "clustertype": 'ExternalManaged', + "username": '', + "password": '', + "url": '', + "clustername": 'VMWare Cluster with Space in DC name', + }, + "upload_volume": { + "diskname": "UploadVol", + "format": "VHD", + "url": "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2", + "checksum": "", + }, + "bootableIso": + { + "displaytext": "Test Bootable ISO", + "name": "testISO", + "bootable": True, "ispublic": False, - "isextractable": True, - "mode": "HTTP_DOWNLOAD", - "templatefilter": "self", - "url": "http://people.apache.org/~sanjeev/systemvm64template-2014-09-30-4.3-vmware.ova", - "hypervisor": "vmware", - "format": "OVA", - "nicadapter": "vmxnet3", - "kvm": { - "url": "" - }, - "vmware": { - "url": "" - }, - "xenserver": { - "url": "" - }, - "hyperv": { - "url": "" - }, - "ostype": 'CentOS 5.3 (64-bit)', + "url": "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso", + "ostype": 'Other Linux (64-bit)', "mode": 'HTTP_DOWNLOAD' - } + }, + "setHostConfigurationForIngressRule": False, + "restartManagementServerThroughTestCase": False, + "vmxnet3template": { + "displaytext": "VMXNET3 Template", + "name": "VMXNET3 template", + "ostype": "CentOS 5.6 (64-bit)", + "isfeatured": True, + "ispublic": False, + "isextractable": True, + "mode": "HTTP_DOWNLOAD", + "templatefilter": "self", + "url": "http://people.apache.org/~sanjeev/systemvm64template-2014-09-30-4.3-vmware.ova", + "hypervisor": "vmware", + "format": "OVA", + "nicadapter": "vmxnet3", + "kvm": { + "url": "" + }, + "vmware": { + "url": "" + }, + "xenserver": { + "url": "" + }, + "hyperv": { + "url": "" + }, + "ostype": 'CentOS 5.3 (64-bit)', + "mode": 'HTTP_DOWNLOAD' } + }, + "cks_kubernetes_versions": { + "1.14.9": { + "semanticversion": "1.14.9", + "url": "http://download.cloudstack.org/cks/setup-1.14.9.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.15.0": { + "semanticversion": "1.15.0", + "url": "http://download.cloudstack.org/cks/setup-1.15.0.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.16.0": { + "semanticversion": "1.16.0", + "url": "http://download.cloudstack.org/cks/setup-1.16.0.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.16.3": { + "semanticversion": "1.16.3", + "url": "http://download.cloudstack.org/cks/setup-1.16.3.iso", + "mincpunumber": 2, + "minmemory": 2048 + } + }, + "cks_templates": { + "kvm": { + "name": "Kubernetes-Service-Template-kvm", + "displaytext": "Kubernetes-Service-Template kvm", + "format": "qcow2", + "hypervisor": "kvm", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True" + }, + "xenserver": { + "name": "Kubernetes-Service-Template-xen", + "displaytext": "Kubernetes-Service-Template xen", + "format": "vhd", + "hypervisor": "xenserver", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True" + }, + "vmware": { + "name": "Kubernetes-Service-Template-vmware", + "displaytext": "Kubernetes-Service-Template vmware", + "format": "ova", + "hypervisor": "vmware", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova", + "requireshvm": "True", + "ispublic": "True", + "details": [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] + } + }, + "cks_service_offering": { + "name": "CKS-Instance", + "displaytext": "CKS Instance", + "cpunumber": 2, + "cpuspeed": 1000, + "memory": 2048 + } }