diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index e970cf69394..1cc525c49e0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import com.cloud.hypervisor.Hypervisor; @@ -36,7 +37,6 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; -import org.apache.commons.lang3.StringUtils; public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker { @@ -122,6 +122,9 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } + if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s", vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 3ec49b1dedc..99da4435c8d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -32,6 +32,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.kubernetes.cluster.KubernetesCluster; @@ -39,12 +40,13 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.nio.TrustAllManager; import com.cloud.utils.ssh.SshHelper; -import org.apache.commons.lang3.StringUtils; public class KubernetesClusterUtil { protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class); + public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short"; + public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, String user, File sshKeyFile, String nodeName) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, @@ -324,4 +326,38 @@ public class KubernetesClusterUtil { } return token.toString().substring(0, 64); } + + public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode, + final String ipAddress, final int port, + final String user, final File sshKeyFile, + final String hostName) { + Pair result = null; + try { + result = SshHelper.sshExecute( + ipAddress, port, + user, sshKeyFile, null, + CLUSTER_NODE_VERSION_COMMAND, + 10000, 10000, 20000); + } catch (Exception e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e); + } + return false; + } + if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) { + return false; + } + String response = result.second(); + boolean clientVersionPresent = false; + boolean serverVersionPresent = false; + for (String line : response.split("\n")) { + if (line.contains("Client Version") && line.contains(String.format("v%s", version))) { + clientVersionPresent = true; + } + if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) { + serverVersionPresent = true; + } + } + return clientVersionPresent && (!isControlNode || serverVersionPresent); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 80ea10df1f0..84f764de93e 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -37,6 +37,9 @@ if [ $# -gt 3 ]; then fi export PATH=$PATH:/opt/bin +if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then + export PATH=$PATH:/usr/sbin +fi ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ @@ -149,4 +152,7 @@ if [ -d "$BINARIES_DIR" ]; then if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" fi +else + echo "ERROR: Unable to access Binaries directory for upgrade version ${UPGRADE_VERSION}" + exit 1 fi diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java new file mode 100644 index 00000000000..475eeba7fc8 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.io.File; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(SshHelper.class) +public class KubernetesClusterUtilTest { + String ipAddress = "10.1.1.1"; + int port = 2222; + String user = "user"; + File sshKeyFile = Mockito.mock(File.class); + String hostName = "host"; + + private void mockSshHelperExecuteThrowAndTestVersionMatch() { + try { + Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class); + } catch (Exception e) { + Assert.fail(String.format("Exception: %s", e.getMessage())); + } + Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName)); + } + + private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) { + try { + Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response)); + } catch (Exception e) { + Assert.fail(String.format("Exception: %s", e.getMessage())); + } + boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void testClusterNodeVersionMatches() { + PowerMockito.mockStatic(SshHelper.class); + String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" + + "The connection to the server localhost:8080 was refused - did you specify the right host or port?"; + String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" + + "Kustomize Version: v4.5.4\n" + + "The connection to the server localhost:8080 was refused - did you specify the right host or port?"; + String v1240ControlNodeOutput = "Client Version: v1.24.0\n" + + "Kustomize Version: v4.5.4\n" + + "Server Version: v1.24.0"; + mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true); + + mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true); + + mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false); + + mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false); + + mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false); + + mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" + + "Kustomize Version: v4.5.4\n" + + "Server Version: v1.23.0", true, false); + + mockSshHelperExecuteAndTestVersionMatch(true, null, false, false); + + mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false); + + mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false); + + mockSshHelperExecuteThrowAndTestVersionMatch(); + } +}