apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CKS: fix upgrade of HA cluster (#7118)

Browse Source
This commit is contained in:
Wei Zhou 2023-01-24 16:11:32 +01:00 committed by GitHub
parent a7d2d8d750
commit 10c77c88c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
View File

@ -122,7 +122,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
}
if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) {
if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) {
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
if (LOGGER.isInfoEnabled()) {

60
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java
View File

@ -45,13 +45,14 @@ public class KubernetesClusterUtil {
protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class);
public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short";
public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'";
public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'";
public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port,
String user, File sshKeyFile, String nodeName) throws Exception {
Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port,
user, sshKeyFile, null,
String.format("sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()),
String.format(CLUSTER_NODE_READY_COMMAND, nodeName.toLowerCase()),
10000, 10000, 20000);
if (result.first() && nodeName.equals(result.second().trim())) {
return true;
@ -327,37 +328,44 @@ public class KubernetesClusterUtil {
return token.toString().substring(0, 64);
}
public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode,
public static boolean clusterNodeVersionMatches(final String version,
final String ipAddress, final int port,
final String user, final File sshKeyFile,
final String hostName) {
Pair<Boolean, String> result = null;
try {
result = SshHelper.sshExecute(
ipAddress, port,
user, sshKeyFile, null,
CLUSTER_NODE_VERSION_COMMAND,
10000, 10000, 20000);
} catch (Exception e) {
final String hostName,
final long timeoutTime, final long waitDuration) {
int retry = 10;
while (System.currentTimeMillis() < timeoutTime && retry-- > 0) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
LOGGER.debug(String.format("Checking if the Kubernetes version of cluster node %s is %s", hostName, version));
}
try {
Pair<Boolean, String> result = SshHelper.sshExecute(
ipAddress, port,
user, sshKeyFile, null,
String.format(CLUSTER_NODE_VERSION_COMMAND, hostName.toLowerCase()),
10000, 10000, 20000);
if (clusterNodeVersionMatches(result, version)) {
return true;
}
} catch (Exception e) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
}
}
try {
Thread.sleep(waitDuration);
} catch (InterruptedException ex) {
LOGGER.warn(String.format("Error while waiting during Kubernetes version check of cluster node : %s", hostName), ex);
}
return false;
}
if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
return false;
}
protected static boolean clusterNodeVersionMatches(final Pair<Boolean, String> result, final String version) {
if (result == null || Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
return false;
}
String response = result.second();
boolean clientVersionPresent = false;
boolean serverVersionPresent = false;
for (String line : response.split("\n")) {
if (line.contains("Client Version") && line.contains(String.format("v%s", version))) {
clientVersionPresent = true;
}
if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) {
serverVersionPresent = true;
}
}
return clientVersionPresent && (!isControlNode || serverVersionPresent);
return response.contains(String.format("v%s", version));
}
}

49
plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java
View File

@ -39,54 +39,29 @@ public class KubernetesClusterUtilTest {
String hostName = "host";
private void mockSshHelperExecuteThrowAndTestVersionMatch() {
try {
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class);
} catch (Exception e) {
Assert.fail(String.format("Exception: %s", e.getMessage()));
}
Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName));
Pair<Boolean, String> resultPair = null;
boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0");
Assert.assertFalse(result);
}
private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) {
try {
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response));
} catch (Exception e) {
Assert.fail(String.format("Exception: %s", e.getMessage()));
}
boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName);
private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean expectedResult) {
Pair<Boolean, String> resultPair = new Pair<>(status, response);
boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0");
Assert.assertEquals(expectedResult, result);
}
@Test
public void testClusterNodeVersionMatches() {
PowerMockito.mockStatic(SshHelper.class);
String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" +
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
String v1240ControlNodeOutput = "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"Server Version: v1.24.0";
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true);
String v1233WorkerNodeOutput = "v1.23.3";
String v1240WorkerNodeOutput = "v1.24.0";
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true);
mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true);
mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false);
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false);
mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false);
mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false);
mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false);
mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"Server Version: v1.23.0", true, false);
mockSshHelperExecuteAndTestVersionMatch(true, null, false, false);
mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false);
mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false);
mockSshHelperExecuteAndTestVersionMatch(false, v1233WorkerNodeOutput, false);
mockSshHelperExecuteThrowAndTestVersionMatch();
}