diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 28f925915c9..e60807ec56d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -142,8 +142,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String joinIpKey = "{{ k8s_master.join_ip }}"; - final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index c6f939cf426..9a30fdd8235 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -89,8 +89,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return kubernetesClusterVersion; } - private Pair> getKubernetesControlIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException { - String controlIp = null; + private Pair> getKubernetesControlNodeIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException { + String controlNodeIp = null; Map requestedIps = null; if (Network.GuestType.Shared.equals(network.getGuestType())) { List vlanIds = new ArrayList<>(); @@ -100,16 +100,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } PublicIp ip = ipAddressManager.getAvailablePublicIpAddressFromVlans(zone.getId(), null, account, Vlan.VlanType.DirectAttached, vlanIds,network.getId(), null, false); if (ip != null) { - controlIp = ip.getAddress().toString(); + controlNodeIp = ip.getAddress().toString(); } requestedIps = new HashMap<>(); Ip ipAddress = ip.getAddress(); boolean isIp6 = ipAddress.isIp6(); requestedIps.put(network.getId(), new Network.IpAddresses(ipAddress.isIp4() ? ip.getAddress().addr() : null, null)); } else { - controlIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); + controlNodeIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); } - return new Pair<>(controlIp, requestedIps); + return new Pair<>(controlNodeIp, requestedIps); } private boolean isKubernetesVersionSupportsHA() { @@ -127,20 +127,20 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return haSupported; } - private String getKubernetesControlConfig(final String controlIp, final String serverIp, - final String hostName, final boolean haSupported, - final boolean ejectIso) throws IOException { - String k8sControlConfig = readResourceFile("/conf/k8s-control-node.yml"); - final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; - final String apiServerKey = "{{ k8s_master.apiserver.key }}"; - final String caCert = "{{ k8s_master.ca.crt }}"; + private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, + final String hostName, final boolean haSupported, + final boolean ejectIso) throws IOException { + String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); + final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; + final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; + final String caCert = "{{ k8s_control_node.ca.crt }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterToken = "{{ k8s_master.cluster.token }}"; - final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; + final String clusterToken = "{{ k8s_control_node.cluster.token }}"; + final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; final List addresses = new ArrayList<>(); - addresses.add(controlIp); - if (!serverIp.equals(controlIp)) { + addresses.add(controlNodeIp); + if (!serverIp.equals(controlNodeIp)) { addresses.add(serverIp); } final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", @@ -149,9 +149,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); - k8sControlConfig = k8sControlConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); - k8sControlConfig = k8sControlConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); - k8sControlConfig = k8sControlConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (!Strings.isNullOrEmpty(sshKeyPair)) { @@ -160,8 +160,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlConfig = k8sControlConfig.replace(sshPubKey, pubKey); - k8sControlConfig = k8sControlConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", @@ -171,9 +171,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); - k8sControlConfig = k8sControlConfig.replace(clusterInitArgsKey, initArgs); - k8sControlConfig = k8sControlConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - return k8sControlConfig; + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + return k8sControlNodeConfig; } private UserVm createKubernetesControlNode(final Network network, String serverIp) throws ManagementServerException, @@ -183,13 +183,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); - Pair> ipAddresses = getKubernetesControlIpAddresses(zone, network, owner); - String controlIp = ipAddresses.first(); + Pair> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner); + String controlNodeIp = ipAddresses.first(); Map requestedIps = ipAddresses.second(); if (Network.GuestType.Shared.equals(network.getGuestType()) && Strings.isNullOrEmpty(serverIp)) { - serverIp = controlIp; + serverIp = controlNodeIp; } - Network.IpAddresses addrs = new Network.IpAddresses(controlIp, null); + Network.IpAddresses addrs = new Network.IpAddresses(controlNodeIp, null); long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { @@ -201,13 +201,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } hostName = getKubernetesClusterNodeAvailableName(hostName); boolean haSupported = isKubernetesVersionSupportsHA(); - String k8sControlConfig = null; + String k8sControlNodeConfig = null; try { - k8sControlConfig = getKubernetesControlConfig(controlIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); } catch (IOException e) { - logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); + logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); } - String base64UserData = Base64.encodeBase64String(k8sControlConfig.getBytes(StringUtils.getPreferredCharset())); + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(StringUtils.getPreferredCharset())); controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), @@ -218,12 +218,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return controlVm; } - private String getKubernetesAdditionalControlConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlConfig = readResourceFile("/conf/k8s-control-node-add.yml"); - final String joinIpKey = "{{ k8s_master.join_ip }}"; - final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { + String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; + final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); @@ -233,12 +233,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlConfig = k8sControlConfig.replace(sshPubKey, pubKey); - k8sControlConfig = k8sControlConfig.replace(joinIpKey, joinIp); - k8sControlConfig = k8sControlConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sControlConfig = k8sControlConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); - k8sControlConfig = k8sControlConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - return k8sControlConfig; + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + return k8sControlNodeConfig; } private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance) throws ManagementServerException, @@ -255,13 +255,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-control-%d", kubernetesClusterNodeNamePrefix, additionalControlNodeInstance + 1)); - String k8sControlConfig = null; + String k8sControlNodeConfig = null; try { - k8sControlConfig = getKubernetesAdditionalControlConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); } - String base64UserData = Base64.encodeBase64String(k8sControlConfig.getBytes(StringUtils.getPreferredCharset())); + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(StringUtils.getPreferredCharset())); additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 682aaaca812..cbfa6accff5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -101,9 +101,9 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle @Param(description = "keypair details") private String keypair; - @Deprecated + @Deprecated(since = "4.16") @SerializedName(ApiConstants.MASTER_NODES) - @Param(description = "the master nodes count for the Kubernetes cluster") + @Param(description = "the master nodes count for the Kubernetes cluster. This parameter is deprecated, please use 'controlnodes' parameter.") private Long masterNodes; @SerializedName(ApiConstants.CONTROL_NODES) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 787ea97491c..a8650ac957b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -196,7 +196,7 @@ write-files: if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin fi - kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification sudo touch /home/core/success echo "true" > /home/core/success @@ -229,7 +229,7 @@ coreos: Type=simple StartLimitInterval=0 Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version + ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version ExecStart=/opt/bin/deploy-kube-system update: diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index db7d7530ed8..c2cecc4a099 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -29,17 +29,17 @@ write-files: - path: /etc/kubernetes/pki/cloudstack/ca.crt permissions: '0644' content: | - {{ k8s_master.ca.crt }} + {{ k8s_control_node.ca.crt }} - path: /etc/kubernetes/pki/cloudstack/apiserver.crt permissions: '0644' content: | - {{ k8s_master.apiserver.crt }} + {{ k8s_control_node.apiserver.crt }} - path: /etc/kubernetes/pki/cloudstack/apiserver.key permissions: '0600' content: | - {{ k8s_master.apiserver.key }} + {{ k8s_control_node.apiserver.key }} - path: /opt/bin/setup-kube-system permissions: 0700 @@ -204,7 +204,7 @@ write-files: fi retval=0 set +e - kubeadm init --token {{ k8s_master.cluster.token }} --token-ttl 0 {{ k8s_master.cluster.initargs }} + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} retval=$? set -e if [ $retval -eq 0 ]; then diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index d2f5454a669..f65cc9c8296 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -196,7 +196,7 @@ write-files: if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin fi - kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification + kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification sudo touch /home/core/success echo "true" > /home/core/success @@ -229,7 +229,7 @@ coreos: Type=simple StartLimitInterval=0 Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version + ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version ExecStart=/opt/bin/deploy-kube-system update: diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index ffd32a2014f..d6ecbf53b96 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -430,7 +430,7 @@ export default { fields.push('zonename') return fields }, - details: ['name', 'description', 'zonename', 'kubernetesversionname', 'size', 'masternodes', 'cpunumber', 'memory', 'keypair', 'associatednetworkname', 'account', 'domain', 'zonename'], + details: ['name', 'description', 'zonename', 'kubernetesversionname', 'size', 'controlnodes', 'cpunumber', 'memory', 'keypair', 'associatednetworkname', 'account', 'domain', 'zonename'], tabs: [{ name: 'k8s', component: () => import('@/views/compute/KubernetesServiceTab.vue')