apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

cks: fix k8s version upgrade (#6513)

Browse Source 
Fixes #6514

On latest systemvm template used for CKS /usr/sbin is not present in the $PATH for normal user used during upgrade. This leads to failure for blkid command. Due to this during k8s version upgrade ISO is not being able to mount on the k8s cluster VMs and upgrade process is not carried out.
This PR fixes mounting of k8s version ISO and also returns failure for script when ISO mounting is failed.
Same failure is not seen during deployment of the cluster because setup-kube-system workflow is executed as ROOT user and it has a different value for $PATH.
From /etc/login.defs:

ENV_SUPATH	PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV_PATH	PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
This commit is contained in:
Abhishek Kumar 2022-07-06 12:34:49 +05:30 committed by GitHub
parent 9e5cda59ce
commit 67e941f690
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 140 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
View File

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import com.cloud.hypervisor.Hypervisor;
@ -36,7 +37,6 @@ import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SshHelper;
import org.apache.commons.lang3.StringUtils;
public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker {
@ -122,6 +122,9 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
}
if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) {
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
if (LOGGER.isInfoEnabled()) {
LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s",
vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid()));

38
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java
View File

@ -32,6 +32,7 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.kubernetes.cluster.KubernetesCluster;
@ -39,12 +40,13 @@ import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.nio.TrustAllManager;
import com.cloud.utils.ssh.SshHelper;
import org.apache.commons.lang3.StringUtils;
public class KubernetesClusterUtil {
protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class);
public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short";
public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port,
String user, File sshKeyFile, String nodeName) throws Exception {
Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port,
@ -324,4 +326,38 @@ public class KubernetesClusterUtil {
}
return token.toString().substring(0, 64);
}
public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode,
final String ipAddress, final int port,
final String user, final File sshKeyFile,
final String hostName) {
Pair<Boolean, String> result = null;
try {
result = SshHelper.sshExecute(
ipAddress, port,
user, sshKeyFile, null,
CLUSTER_NODE_VERSION_COMMAND,
10000, 10000, 20000);
} catch (Exception e) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
}
return false;
}
if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
return false;
}
String response = result.second();
boolean clientVersionPresent = false;
boolean serverVersionPresent = false;
for (String line : response.split("\n")) {
if (line.contains("Client Version") && line.contains(String.format("v%s", version))) {
clientVersionPresent = true;
}
if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) {
serverVersionPresent = true;
}
}
return clientVersionPresent && (!isControlNode || serverVersionPresent);
}
}

6
plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
View File

@ -37,6 +37,9 @@ if [ $# -gt 3 ]; then
fi
export PATH=$PATH:/opt/bin
if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
export PATH=$PATH:/usr/sbin
fi
ISO_MOUNT_DIR=/mnt/k8sdisk
BINARIES_DIR=${ISO_MOUNT_DIR}/
@ -149,4 +152,7 @@ if [ -d "$BINARIES_DIR" ]; then
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
fi
else
echo "ERROR: Unable to access Binaries directory for upgrade version ${UPGRADE_VERSION}"
exit 1
fi

93
plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java Normal file
View File

@ -0,0 +1,93 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.utils;
import java.io.File;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.cloud.utils.Pair;
import com.cloud.utils.ssh.SshHelper;
@RunWith(PowerMockRunner.class)
@PrepareForTest(SshHelper.class)
public class KubernetesClusterUtilTest {
String ipAddress = "10.1.1.1";
int port = 2222;
String user = "user";
File sshKeyFile = Mockito.mock(File.class);
String hostName = "host";
private void mockSshHelperExecuteThrowAndTestVersionMatch() {
try {
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class);
} catch (Exception e) {
Assert.fail(String.format("Exception: %s", e.getMessage()));
}
Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName));
}
private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) {
try {
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response));
} catch (Exception e) {
Assert.fail(String.format("Exception: %s", e.getMessage()));
}
boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName);
Assert.assertEquals(expectedResult, result);
}
@Test
public void testClusterNodeVersionMatches() {
PowerMockito.mockStatic(SshHelper.class);
String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" +
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
String v1240ControlNodeOutput = "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"Server Version: v1.24.0";
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true);
mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true);
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false);
mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false);
mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false);
mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" +
"Kustomize Version: v4.5.4\n" +
"Server Version: v1.23.0", true, false);
mockSshHelperExecuteAndTestVersionMatch(true, null, false, false);
mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false);
mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false);
mockSshHelperExecuteThrowAndTestVersionMatch();
}
}