mirror of
https://github.com/apache/cloudstack.git
synced 2025-11-03 04:12:31 +01:00
Support unstacked ETCD
--------- Co-authored-by: nvazquez <nicovazquez90@gmail.com>
This commit is contained in:
Pearl Dsilva
nvazquez
parent
b72a7cd020
commit
1e40aed085
@ -120,6 +120,7 @@ public class ApiConstants {
|
||||
public static final String ENCRYPT_FORMAT = "encryptformat";
|
||||
public static final String ENCRYPT_ROOT = "encryptroot";
|
||||
public static final String ENCRYPTION_SUPPORTED = "encryptionsupported";
|
||||
public static final String ETCD_IPS = "etcdips";
|
||||
public static final String MIN_IOPS = "miniops";
|
||||
public static final String MAX_IOPS = "maxiops";
|
||||
public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve";
|
||||
@ -1055,6 +1056,7 @@ public class ApiConstants {
|
||||
public static final String ETCD_NODES = "etcdnodes";
|
||||
public static final String EXTERNAL_NODES = "externalnodes";
|
||||
public static final String IS_EXTERNAL_NODE = "isexternalnode";
|
||||
public static final String IS_ETCD_NODE = "isetcdnode";
|
||||
public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion";
|
||||
public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid";
|
||||
public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize";
|
||||
|
||||
@ -30,11 +30,15 @@ public class KubernetesUserVmResponse extends UserVmResponse {
|
||||
@Param(description = "If the VM is an externally added node")
|
||||
private boolean isExternalNode;
|
||||
|
||||
public boolean isExternalNode() {
|
||||
return isExternalNode;
|
||||
}
|
||||
@SerializedName(ApiConstants.IS_ETCD_NODE)
|
||||
@Param(description = "If the VM is an etcd node")
|
||||
private boolean isEtcdNode;
|
||||
|
||||
public void setExternalNode(boolean externalNode) {
|
||||
isExternalNode = externalNode;
|
||||
}
|
||||
|
||||
public void setEtcdNode(boolean etcdNode) {
|
||||
isEtcdNode = etcdNode;
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,18 +42,23 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import com.cloud.dc.DedicatedResourceVO;
|
||||
import com.cloud.dc.dao.DedicatedResourceDao;
|
||||
import com.cloud.exception.ManagementServerException;
|
||||
import com.cloud.exception.ResourceUnavailableException;
|
||||
import com.cloud.host.Host;
|
||||
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
|
||||
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker;
|
||||
import com.cloud.network.dao.NsxProviderDao;
|
||||
import com.cloud.network.element.NsxProviderVO;
|
||||
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker;
|
||||
import com.cloud.network.rules.PortForwardingRuleVO;
|
||||
import com.cloud.network.rules.dao.PortForwardingRulesDao;
|
||||
import com.cloud.template.TemplateApiService;
|
||||
import com.cloud.user.dao.AccountDao;
|
||||
import com.cloud.uservm.UserVm;
|
||||
@ -304,6 +309,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
|
||||
private UserVmService userVmService;
|
||||
@Inject
|
||||
private TemplateApiService templateService;
|
||||
@Inject
|
||||
private PortForwardingRulesDao pfRuleDao;
|
||||
|
||||
private void logMessage(final Level logLevel, final String message, final Exception e) {
|
||||
if (logLevel == Level.WARN) {
|
||||
@ -745,10 +752,23 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
|
||||
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response");
|
||||
}
|
||||
kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode());
|
||||
kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode());
|
||||
vmResponses.add(kubernetesUserVmResponse);
|
||||
}
|
||||
}
|
||||
response.setExternalNodes(vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).count());
|
||||
List<Long> etcdNodeIds = vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());
|
||||
List<Long> etcdIpIds = new ArrayList<>();
|
||||
Map<String, String> etcdIps = new HashMap<>();
|
||||
int etcdNodeSshPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value();
|
||||
etcdNodeIds.forEach(id -> {
|
||||
etcdIpIds.addAll(pfRuleDao.listByVm(id).stream().filter(rule -> rule.getSourcePortStart() == etcdNodeSshPort)
|
||||
.map(PortForwardingRuleVO::getSourceIpAddressId).collect(Collectors.toList()));
|
||||
});
|
||||
etcdIpIds.forEach(id -> {
|
||||
IPAddressVO ipAddress = ipAddressDao.findById(id);
|
||||
etcdIps.put(ipAddress.getUuid(), ipAddress.getAddress().addr());
|
||||
});
|
||||
response.setEtcdIps(etcdIps);
|
||||
}
|
||||
response.setHasAnnotation(annotationDao.hasAnnotations(kubernetesCluster.getUuid(),
|
||||
AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), accountService.isRootAdmin(caller.getId())));
|
||||
@ -758,6 +778,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
|
||||
response.setMaxSize(kubernetesCluster.getMaxSize());
|
||||
response.setClusterType(kubernetesCluster.getClusterType());
|
||||
response.setCreated(kubernetesCluster.getCreated());
|
||||
|
||||
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@ -1554,7 +1577,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
|
||||
*/
|
||||
|
||||
@Override
|
||||
public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException {
|
||||
public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate)
|
||||
throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException {
|
||||
if (!KubernetesServiceEnabled.value()) {
|
||||
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
|
||||
}
|
||||
@ -2361,7 +2385,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
|
||||
KubernetesControlNodeInstallAttemptWait,
|
||||
KubernetesControlNodeInstallReattempts,
|
||||
KubernetesWorkerNodeInstallAttemptWait,
|
||||
KubernetesWorkerNodeInstallReattempts
|
||||
KubernetesWorkerNodeInstallReattempts,
|
||||
KubernetesEtcdNodeStartPort
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,6 +16,9 @@
|
||||
// under the License.
|
||||
package com.cloud.kubernetes.cluster;
|
||||
|
||||
import com.cloud.exception.InsufficientCapacityException;
|
||||
import com.cloud.exception.ManagementServerException;
|
||||
import com.cloud.exception.ResourceUnavailableException;
|
||||
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd;
|
||||
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
|
||||
import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
|
||||
@ -129,6 +132,12 @@ public interface KubernetesClusterService extends PluggableService, Configurable
|
||||
"Number of times the offline installation of K8S will be re-attempted",
|
||||
true,
|
||||
KubernetesServiceEnabled.key());
|
||||
static final ConfigKey<Integer> KubernetesEtcdNodeStartPort = new ConfigKey<Integer>("Advanced", Integer.class,
|
||||
"cloud.kubernetes.etcd.node.start.port",
|
||||
"50000",
|
||||
"Start port for Port forwarding rules for etcd nodes",
|
||||
true,
|
||||
KubernetesServiceEnabled.key());
|
||||
|
||||
KubernetesCluster findById(final Long id);
|
||||
|
||||
@ -136,7 +145,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable
|
||||
|
||||
KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException;
|
||||
|
||||
boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException;
|
||||
boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException;
|
||||
|
||||
boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudRuntimeException;
|
||||
|
||||
|
||||
@ -133,6 +133,8 @@ public class KubernetesClusterActionWorker {
|
||||
public static final int CLUSTER_API_PORT = 6443;
|
||||
public static final int DEFAULT_SSH_PORT = 22;
|
||||
public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222;
|
||||
public static final int ETCD_NODE_CLIENT_REQUEST_PORT = 2379;
|
||||
public static final int ETCD_NODE_PEER_COMM_PORT = 2380;
|
||||
public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT;
|
||||
|
||||
public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
|
||||
@ -358,14 +360,16 @@ public class KubernetesClusterActionWorker {
|
||||
return new File(keyFile);
|
||||
}
|
||||
|
||||
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode,
|
||||
boolean isExternalNode, boolean markForManualUpgrade) {
|
||||
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId,
|
||||
boolean isControlNode, boolean isExternalNode,
|
||||
boolean isEtcdNode, boolean markForManualUpgrade) {
|
||||
return Transaction.execute(new TransactionCallback<KubernetesClusterVmMapVO>() {
|
||||
@Override
|
||||
public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) {
|
||||
KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode);
|
||||
newClusterVmMap.setExternalNode(isExternalNode);
|
||||
newClusterVmMap.setManualUpgrade(markForManualUpgrade);
|
||||
newClusterVmMap.setEtcdNode(isEtcdNode);
|
||||
kubernetesClusterVmMapDao.persist(newClusterVmMap);
|
||||
return newClusterVmMap;
|
||||
}
|
||||
@ -436,7 +440,7 @@ public class KubernetesClusterActionWorker {
|
||||
return publicIp;
|
||||
}
|
||||
|
||||
protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws
|
||||
protected IpAddress acquireVpcTierKubernetesPublicIp(Network network, boolean forEtcd) throws
|
||||
InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
|
||||
IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null);
|
||||
if (ip == null) {
|
||||
@ -444,7 +448,19 @@ public class KubernetesClusterActionWorker {
|
||||
}
|
||||
ip = vpcService.associateIPToVpc(ip.getId(), network.getVpcId());
|
||||
ip = ipAddressManager.associateIPToGuestNetwork(ip.getId(), network.getId(), false);
|
||||
kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false);
|
||||
if (!forEtcd) {
|
||||
kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false);
|
||||
}
|
||||
return ip;
|
||||
}
|
||||
|
||||
protected IpAddress acquirePublicIpForIsolatedNetwork(Network network) throws
|
||||
InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
|
||||
IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null);
|
||||
if (ip == null) {
|
||||
return null;
|
||||
}
|
||||
ip = networkService.associateIPToNetwork(ip.getId(), network.getId());
|
||||
return ip;
|
||||
}
|
||||
|
||||
@ -476,7 +492,7 @@ public class KubernetesClusterActionWorker {
|
||||
return new Pair<>(address.getAddress().addr(), port);
|
||||
}
|
||||
if (acquireNewPublicIpForVpcTierIfNeeded) {
|
||||
address = acquireVpcTierKubernetesPublicIp(network);
|
||||
address = acquireVpcTierKubernetesPublicIp(network, false);
|
||||
if (address != null) {
|
||||
return new Pair<>(address.getAddress().addr(), port);
|
||||
}
|
||||
|
||||
@ -264,7 +264,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
|
||||
kubernetesClusterDao.update(clusterId, kubernetesClusterVO);
|
||||
kubernetesCluster = kubernetesClusterVO;
|
||||
|
||||
finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, manualUpgrade));
|
||||
finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, false, manualUpgrade));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -335,7 +335,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
|
||||
List<UserVm> nodes = new ArrayList<>();
|
||||
for (int i = offset + 1; i <= nodeCount; i++) {
|
||||
UserVm vm = createKubernetesNode(publicIpAddress, domainId, accountId);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false, false);
|
||||
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
|
||||
resizeNodeVolume(vm);
|
||||
}
|
||||
@ -778,6 +778,21 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
|
||||
return prefix;
|
||||
}
|
||||
|
||||
protected String getEtcdNodeNameForCluster() {
|
||||
String prefix = kubernetesCluster.getName();
|
||||
if (!NetUtils.verifyDomainNameLabel(prefix, true)) {
|
||||
prefix = prefix.replaceAll("[^a-zA-Z0-9-]", "");
|
||||
if (prefix.isEmpty()) {
|
||||
prefix = kubernetesCluster.getUuid();
|
||||
}
|
||||
}
|
||||
prefix = "etcd-" + prefix;
|
||||
if (prefix.length() > 40) {
|
||||
prefix = prefix.substring(0, 40);
|
||||
}
|
||||
return prefix;
|
||||
}
|
||||
|
||||
protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, final Long size,
|
||||
final Long serviceOfferingId, final Boolean autoscaleEnabled,
|
||||
final Long minSize, final Long maxSize,
|
||||
|
||||
@ -31,6 +31,8 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.cloud.exception.NetworkRuleConflictException;
|
||||
import com.cloud.utils.Ternary;
|
||||
import org.apache.cloudstack.api.BaseCmd;
|
||||
import org.apache.cloudstack.api.InternalIdentity;
|
||||
import org.apache.cloudstack.framework.ca.Certificate;
|
||||
@ -78,6 +80,7 @@ import com.cloud.vm.VmDetailConstants;
|
||||
import org.apache.logging.log4j.Level;
|
||||
|
||||
import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL;
|
||||
import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD;
|
||||
|
||||
public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker {
|
||||
|
||||
@ -133,7 +136,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
}
|
||||
|
||||
private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
|
||||
final String hostName, final boolean haSupported,
|
||||
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
|
||||
final boolean ejectIso) throws IOException {
|
||||
String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml");
|
||||
final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
|
||||
@ -145,12 +148,20 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
final String ejectIsoKey = "{{ k8s.eject.iso }}";
|
||||
final String installWaitTime = "{{ k8s.install.wait.time }}";
|
||||
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
|
||||
final String externalEtcdNodes = "{{ etcd.unstacked_etcd }}";
|
||||
final String etcdEndpointList = "{{ etcd.etcd_endpoint_list }}";
|
||||
final String k8sServerIp = "{{ k8s_control.server_ip }}";
|
||||
final String k8sApiPort = "{{ k8s.api_server_port }}";
|
||||
final String certSans = "{{ k8s_control.server_ips }}";
|
||||
final String k8sCertificate = "{{ k8s_control.certificate_key }}";
|
||||
|
||||
final List<String> addresses = new ArrayList<>();
|
||||
addresses.add(controlNodeIp);
|
||||
if (!serverIp.equals(controlNodeIp)) {
|
||||
addresses.add(serverIp);
|
||||
}
|
||||
|
||||
boolean externalEtcd = !etcdIps.isEmpty();
|
||||
final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes",
|
||||
"kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"),
|
||||
addresses, 3650, null);
|
||||
@ -159,6 +170,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates());
|
||||
final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
|
||||
final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
|
||||
String endpointList = getEtcdEndpointList(etcdIps);
|
||||
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n "));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n "));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n "));
|
||||
@ -174,6 +187,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(externalEtcdNodes, String.valueOf(externalEtcd));
|
||||
String initArgs = "";
|
||||
if (haSupported) {
|
||||
initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ",
|
||||
@ -185,12 +199,18 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion());
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs);
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(etcdEndpointList, endpointList);
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sServerIp, serverIp);
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sApiPort, String.valueOf(CLUSTER_API_PORT));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp));
|
||||
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
|
||||
|
||||
k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
|
||||
|
||||
return k8sControlNodeConfig;
|
||||
}
|
||||
|
||||
private UserVm createKubernetesControlNode(final Network network, String serverIp, Long domainId, Long accountId) throws ManagementServerException,
|
||||
private UserVm createKubernetesControlNode(final Network network, String serverIp, List<Network.IpAddresses> etcdIps, Long domainId, Long accountId) throws ManagementServerException,
|
||||
ResourceUnavailableException, InsufficientCapacityException {
|
||||
UserVm controlVm = null;
|
||||
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
|
||||
@ -217,7 +237,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
boolean haSupported = isKubernetesVersionSupportsHA();
|
||||
String k8sControlNodeConfig = null;
|
||||
try {
|
||||
k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
|
||||
k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
|
||||
} catch (IOException e) {
|
||||
logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
|
||||
}
|
||||
@ -281,6 +301,78 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
return k8sControlNodeConfig;
|
||||
}
|
||||
|
||||
private String getInitialEtcdClusterDetails(List<String> ipAddresses, List<String> hostnames) {
|
||||
String initialCluster = "%s=http://%s:2380";
|
||||
StringBuilder clusterInfo = new StringBuilder();
|
||||
for (int i = 0; i < ipAddresses.size(); i++) {
|
||||
clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i)));
|
||||
if (i < ipAddresses.size()-1) {
|
||||
clusterInfo.append(",");
|
||||
}
|
||||
}
|
||||
return clusterInfo.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param ipAddresses list of etcd node guest IPs
|
||||
* @return a formatted list of etcd endpoints adhering to YAML syntax
|
||||
*/
|
||||
private String getEtcdEndpointList(List<Network.IpAddresses> ipAddresses) {
|
||||
StringBuilder endpoints = new StringBuilder();
|
||||
for (int i = 0; i < ipAddresses.size(); i++) {
|
||||
endpoints.append(String.format("- http://%s:2379", ipAddresses.get(i).getIp4Address()));
|
||||
if (i < ipAddresses.size()-1) {
|
||||
endpoints.append("\n ");
|
||||
}
|
||||
}
|
||||
return endpoints.toString();
|
||||
}
|
||||
|
||||
|
||||
private List<String> getEtcdNodeHostnames() {
|
||||
List<String> hostnames = new ArrayList<>();
|
||||
for (int etcdNodeIndex = 0; etcdNodeIndex <= kubernetesCluster.getEtcdNodeCount(); etcdNodeIndex++) {
|
||||
String suffix = Long.toHexString(System.currentTimeMillis());
|
||||
hostnames.add(String.format("%s-%s-%s", getEtcdNodeNameForCluster(), etcdNodeIndex, suffix));
|
||||
}
|
||||
return hostnames;
|
||||
}
|
||||
|
||||
private String getEtcdNodeConfig(final List<String> ipAddresses, final List<String> hostnames, final int etcdNodeIndex,
|
||||
final boolean ejectIso) throws IOException {
|
||||
String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml");
|
||||
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
|
||||
final String ejectIsoKey = "{{ k8s.eject.iso }}";
|
||||
final String installWaitTime = "{{ k8s.install.wait.time }}";
|
||||
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
|
||||
final String etcdNodeName = "{{ etcd.node_name }}";
|
||||
final String etcdNodeIp = "{{ etcd.node_ip }}";
|
||||
final String etcdInitialClusterNodes = "{{ etcd.initial_cluster_nodes }}";
|
||||
|
||||
final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
|
||||
final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
|
||||
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
|
||||
String sshKeyPair = kubernetesCluster.getKeyPair();
|
||||
if (StringUtils.isNotEmpty(sshKeyPair)) {
|
||||
SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair);
|
||||
if (sshkp != null) {
|
||||
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
|
||||
}
|
||||
}
|
||||
String initialClusterDetails = getInitialEtcdClusterDetails(ipAddresses, hostnames);
|
||||
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installWaitTime, String.valueOf(waitTime));
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(sshPubKey, pubKey);
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeName, hostnames.get(etcdNodeIndex));
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeIp, ipAddresses.get(etcdNodeIndex));
|
||||
k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdInitialClusterNodes, initialClusterDetails);
|
||||
|
||||
return k8sEtcdNodeConfig;
|
||||
}
|
||||
|
||||
private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance,
|
||||
final Long domainId, final Long accountId) throws ManagementServerException,
|
||||
ResourceUnavailableException, InsufficientCapacityException {
|
||||
@ -336,12 +428,56 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
return additionalControlVm;
|
||||
}
|
||||
|
||||
private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress,
|
||||
private UserVm createEtcdNode(List<Network.IpAddresses> requestedIps, List<String> etcdNodeHostnames, int etcdNodeIndex, Long domainId, Long accountId) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException {
|
||||
UserVm etcdNode = null;
|
||||
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
|
||||
ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(ETCD, kubernetesCluster);
|
||||
List<Long> networkIds = Collections.singletonList(kubernetesCluster.getNetworkId());
|
||||
Network.IpAddresses addrs = new Network.IpAddresses(null, null);
|
||||
List<String> guestIps = requestedIps.stream().map(Network.IpAddresses::getIp4Address).collect(Collectors.toList());
|
||||
String k8sControlNodeConfig = null;
|
||||
try {
|
||||
k8sControlNodeConfig = getEtcdNodeConfig(guestIps, etcdNodeHostnames, etcdNodeIndex, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
|
||||
} catch (IOException e) {
|
||||
logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
|
||||
}
|
||||
|
||||
String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
|
||||
List<String> keypairs = new ArrayList<String>();
|
||||
if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
|
||||
keypairs.add(kubernetesCluster.getKeyPair());
|
||||
}
|
||||
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
|
||||
String hostName = etcdNodeHostnames.get(etcdNodeIndex);
|
||||
Map<String, String> customParameterMap = new HashMap<String, String>();
|
||||
if (zone.isSecurityGroupEnabled()) {
|
||||
List<Long> securityGroupIds = new ArrayList<>();
|
||||
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
|
||||
etcdNode = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, securityGroupIds, owner,
|
||||
hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
|
||||
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ?
|
||||
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
|
||||
null, true, null, null);
|
||||
} else {
|
||||
etcdNode = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, owner,
|
||||
hostName, hostName, null, null, null,
|
||||
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
|
||||
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ?
|
||||
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
|
||||
}
|
||||
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info(String.format("Created control VM ID : %s, %s in the Kubernetes cluster : %s", etcdNode.getUuid(), hostName, kubernetesCluster.getName()));
|
||||
}
|
||||
return etcdNode;
|
||||
}
|
||||
|
||||
private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress, final List<Network.IpAddresses> etcdIps,
|
||||
final Long domainId, final Long accountId) throws
|
||||
ManagementServerException, InsufficientCapacityException, ResourceUnavailableException {
|
||||
UserVm k8sControlVM = null;
|
||||
k8sControlVM = createKubernetesControlNode(network, publicIpAddress, domainId, accountId);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false);
|
||||
k8sControlVM = createKubernetesControlNode(network, publicIpAddress, etcdIps, domainId, accountId);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false, false);
|
||||
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
|
||||
resizeNodeVolume(k8sControlVM);
|
||||
}
|
||||
@ -356,14 +492,15 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
return k8sControlVM;
|
||||
}
|
||||
|
||||
private List<UserVm> provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress, final Long domainId, final Long accountId) throws
|
||||
private List<UserVm> provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress, final Long domainId,
|
||||
final Long accountId) throws
|
||||
InsufficientCapacityException, ManagementServerException, ResourceUnavailableException {
|
||||
List<UserVm> additionalControlVms = new ArrayList<>();
|
||||
if (kubernetesCluster.getControlNodeCount() > 1) {
|
||||
for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) {
|
||||
UserVm vm = null;
|
||||
vm = createKubernetesAdditionalControlNode(publicIpAddress, i, domainId, accountId);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false, false);
|
||||
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
|
||||
resizeNodeVolume(vm);
|
||||
}
|
||||
@ -381,6 +518,55 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
return additionalControlVms;
|
||||
}
|
||||
|
||||
private Ternary<List<UserVm>, List<Network.IpAddresses>, List<IpAddress>> provisionEtcdCluster(final Network network, final Long domainId, final Long accountId)
|
||||
throws InsufficientCapacityException, ResourceUnavailableException, ManagementServerException {
|
||||
List<UserVm> etcdNodeVms = new ArrayList<>();
|
||||
List<IpAddress> etcdNodeIps = getEtcdNodePublicIpAddresses(network, kubernetesCluster.getEtcdNodeCount());
|
||||
List<Network.IpAddresses> etcdNodeGuestIps = getEtcdNodeGuestIps(network, kubernetesCluster.getEtcdNodeCount());
|
||||
List<String> etcdHostnames = getEtcdNodeHostnames();
|
||||
for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) {
|
||||
IpAddress ip = etcdNodeIps.get(i);
|
||||
if (Objects.isNull(ip)) {
|
||||
String errMsg = String.format("No public IP found for the network: %s, to create Etcd node for " +
|
||||
"Kubernetes cluster: %s", network, kubernetesCluster.getName());
|
||||
LOGGER.error(errMsg);
|
||||
logAndThrow(Level.ERROR, errMsg);
|
||||
}
|
||||
UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId);
|
||||
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true);
|
||||
startKubernetesVM(vm, domainId, accountId);
|
||||
vm = userVmDao.findById(vm.getId());
|
||||
if (vm == null) {
|
||||
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
|
||||
}
|
||||
etcdNodeVms.add(vm);
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info(String.format("Provisioned additional control VM : %s in to the Kubernetes cluster : %s", vm.getDisplayName(), kubernetesCluster.getName()));
|
||||
}
|
||||
}
|
||||
return new Ternary<>(etcdNodeVms, etcdNodeGuestIps, etcdNodeIps);
|
||||
}
|
||||
|
||||
private List<IpAddress> getEtcdNodePublicIpAddresses(final Network network, final long etcdNodeCount) throws InsufficientAddressCapacityException, ResourceUnavailableException, ResourceAllocationException {
|
||||
List<IpAddress> ipAddresses = new ArrayList<>();
|
||||
for (int i = 1; i <= etcdNodeCount; i++) {
|
||||
if (network.getVpcId() == null) {
|
||||
ipAddresses.add(acquirePublicIpForIsolatedNetwork(network));
|
||||
} else {
|
||||
ipAddresses.add(acquireVpcTierKubernetesPublicIp(network, true));
|
||||
}
|
||||
}
|
||||
return ipAddresses;
|
||||
}
|
||||
|
||||
private List<Network.IpAddresses> getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) {
|
||||
List<Network.IpAddresses> guestIps = new ArrayList<>();
|
||||
for (int i = 1; i <= etcdNodeCount; i++) {
|
||||
guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null));
|
||||
}
|
||||
return guestIps;
|
||||
}
|
||||
|
||||
private Network startKubernetesClusterNetwork(final DeployDestination destination) throws ManagementServerException {
|
||||
final ReservationContext context = new ReservationContextImpl(null, null, null, owner);
|
||||
Network network = networkDao.findById(kubernetesCluster.getNetworkId());
|
||||
@ -428,6 +614,29 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true);
|
||||
}
|
||||
|
||||
protected void setupKubernetesEtcdNetworkRules(List<IpAddress> etcdNodeIps, List<UserVm> etcdVms, Network network) throws ManagementServerException, ResourceUnavailableException {
|
||||
if (!Network.GuestType.Isolated.equals(network.getGuestType())) {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
|
||||
}
|
||||
}
|
||||
List<Long> etcdVmIds = etcdVms.stream().map(UserVm::getId).collect(Collectors.toList());
|
||||
Integer startPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value();
|
||||
for (int i = 0; i < etcdVmIds.size(); i++) {
|
||||
IpAddress publicIp = etcdNodeIps.get(i);
|
||||
try {
|
||||
provisionFirewallRules(publicIp, owner, startPort, startPort);
|
||||
provisionFirewallRules(publicIp, owner, ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_PEER_COMM_PORT);
|
||||
} catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException |
|
||||
NetworkRuleConflictException e) {
|
||||
throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
|
||||
}
|
||||
provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), startPort, DEFAULT_SSH_PORT);
|
||||
provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_CLIENT_REQUEST_PORT);
|
||||
provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), ETCD_NODE_PEER_COMM_PORT, ETCD_NODE_PEER_COMM_PORT);
|
||||
}
|
||||
}
|
||||
|
||||
private void startKubernetesClusterVMs(Long domainId, Long accountId) {
|
||||
List <UserVm> clusterVms = getKubernetesClusterVMs();
|
||||
for (final UserVm vm : clusterVms) {
|
||||
@ -490,7 +699,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO);
|
||||
}
|
||||
|
||||
public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId) {
|
||||
public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException {
|
||||
init();
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info(String.format("Starting Kubernetes cluster : %s", kubernetesCluster.getName()));
|
||||
@ -526,10 +735,20 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
launchPermissionDao.persist(launchPermission);
|
||||
}
|
||||
|
||||
List<UserVm> etcdVms = new ArrayList<>();
|
||||
List<Network.IpAddresses> etcdGuestNodeIps = new ArrayList<>();
|
||||
List<IpAddress> etcdPublicNodeIps = new ArrayList<>();
|
||||
if (kubernetesCluster.getEtcdNodeCount() > 0) {
|
||||
Ternary<List<UserVm>, List<Network.IpAddresses>, List<IpAddress>> etcdNodesAndIps = provisionEtcdCluster(network, domainId, accountId);
|
||||
etcdVms = etcdNodesAndIps.first();
|
||||
etcdGuestNodeIps = etcdNodesAndIps.second();
|
||||
etcdPublicNodeIps = etcdNodesAndIps.third();
|
||||
}
|
||||
|
||||
List<UserVm> clusterVMs = new ArrayList<>();
|
||||
UserVm k8sControlVM = null;
|
||||
try {
|
||||
k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress, domainId, accountId);
|
||||
k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress, etcdGuestNodeIps, domainId, accountId);
|
||||
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
|
||||
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
|
||||
}
|
||||
@ -561,6 +780,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
} catch (ManagementServerException e) {
|
||||
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
|
||||
}
|
||||
try {
|
||||
setupKubernetesEtcdNetworkRules(etcdPublicNodeIps, etcdVms, network);
|
||||
} catch (ManagementServerException e) {
|
||||
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules for etcd nodes", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
|
||||
}
|
||||
attachIsoKubernetesVMs(etcdVms);
|
||||
attachIsoKubernetesVMs(clusterVMs);
|
||||
if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) {
|
||||
String msg = String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to access control node VMs of the cluster", kubernetesCluster.getName());
|
||||
|
||||
@ -217,8 +217,8 @@ public class KubernetesClusterUtil {
|
||||
user, sshKeyFile, null,
|
||||
"sudo /opt/bin/kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l",
|
||||
10000, 10000, 20000);
|
||||
if (result.first()) {
|
||||
return Integer.parseInt(result.second().trim().replace("\"", ""));
|
||||
if (Boolean.TRUE.equals(result.first())) {
|
||||
return Integer.parseInt(result.second().trim().replace("\"", "")) + kubernetesCluster.getEtcdNodeCount().intValue();
|
||||
} else {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster : %s. Output: %s", kubernetesCluster.getName(), result.second()));
|
||||
|
||||
@ -21,7 +21,10 @@ import java.util.Map;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import com.cloud.exception.InsufficientCapacityException;
|
||||
import com.cloud.exception.InvalidParameterValueException;
|
||||
import com.cloud.exception.ManagementServerException;
|
||||
import com.cloud.exception.ResourceUnavailableException;
|
||||
import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
|
||||
import org.apache.cloudstack.acl.RoleType;
|
||||
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
|
||||
@ -328,7 +331,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
|
||||
KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId());
|
||||
response.setResponseName(getCommandName());
|
||||
setResponseObject(response);
|
||||
} catch (CloudRuntimeException e) {
|
||||
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
|
||||
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,6 +18,9 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import com.cloud.exception.InsufficientCapacityException;
|
||||
import com.cloud.exception.ManagementServerException;
|
||||
import com.cloud.exception.ResourceUnavailableException;
|
||||
import com.cloud.user.Account;
|
||||
import org.apache.cloudstack.acl.RoleType;
|
||||
import org.apache.cloudstack.api.APICommand;
|
||||
@ -116,7 +119,8 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd {
|
||||
final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId());
|
||||
response.setResponseName(getCommandName());
|
||||
setResponseObject(response);
|
||||
} catch (CloudRuntimeException ex) {
|
||||
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException |
|
||||
InsufficientCapacityException ex) {
|
||||
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@ package org.apache.cloudstack.api.response;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.cloudstack.api.ApiConstants;
|
||||
import org.apache.cloudstack.api.BaseResponseWithAnnotations;
|
||||
@ -86,10 +87,6 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
|
||||
@Param(description = "the number of the etcd nodes on the Kubernetes cluster")
|
||||
private Long etcdNodes;
|
||||
|
||||
@SerializedName(ApiConstants.EXTERNAL_NODES)
|
||||
@Param(description = "the number of the externally added worker nodes to the Kubernetes cluster")
|
||||
private Long externalNodes;
|
||||
|
||||
@SerializedName(ApiConstants.TEMPLATE_ID)
|
||||
@Param(description = "the ID of the template of the Kubernetes cluster")
|
||||
private String templateId;
|
||||
@ -179,6 +176,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
|
||||
@Param(description = "Public IP Address ID of the cluster")
|
||||
private String ipAddressId;
|
||||
|
||||
@SerializedName(ApiConstants.ETCD_IPS)
|
||||
@Param(description = "Public IP Addresses of the etcd nodes")
|
||||
private Map<String, String> etcdIps;
|
||||
|
||||
@SerializedName(ApiConstants.AUTOSCALING_ENABLED)
|
||||
@Param(description = "Whether autoscaling is enabled for the cluster")
|
||||
private boolean isAutoscalingEnabled;
|
||||
@ -447,14 +448,6 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
|
||||
this.etcdNodes = etcdNodes;
|
||||
}
|
||||
|
||||
public Long getExternalNodes() {
|
||||
return externalNodes;
|
||||
}
|
||||
|
||||
public void setExternalNodes(Long externalNodes) {
|
||||
this.externalNodes = externalNodes;
|
||||
}
|
||||
|
||||
public void setVirtualMachines(List<KubernetesUserVmResponse> virtualMachines) {
|
||||
this.virtualMachines = virtualMachines;
|
||||
}
|
||||
@ -471,6 +464,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
|
||||
this.ipAddressId = ipAddressId;
|
||||
}
|
||||
|
||||
public void setEtcdIps(Map<String, String> etcdIps) {
|
||||
this.etcdIps = etcdIps;
|
||||
}
|
||||
|
||||
public void setAutoscalingEnabled(boolean isAutoscalingEnabled) {
|
||||
this.isAutoscalingEnabled = isAutoscalingEnabled;
|
||||
}
|
||||
|
||||
@ -0,0 +1,134 @@
|
||||
#cloud-config
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
---
|
||||
users:
|
||||
- name: cloud
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/bash
|
||||
ssh_authorized_keys:
|
||||
{{ k8s.ssh.pub.key }}
|
||||
|
||||
write_files:
|
||||
- path: /opt/bin/setup-etcd-node
|
||||
permissions: '0700'
|
||||
owner: root:root
|
||||
content: |
|
||||
#!/bin/bash -e
|
||||
|
||||
if [[ -f "/home/cloud/success" ]]; then
|
||||
echo "Already provisioned!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ISO_MOUNT_DIR=/mnt/etcddisk
|
||||
BINARIES_DIR=${ISO_MOUNT_DIR}/
|
||||
ATTEMPT_ONLINE_INSTALL=false
|
||||
setup_complete=false
|
||||
|
||||
OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
|
||||
MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
|
||||
if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
|
||||
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
|
||||
fi
|
||||
if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then
|
||||
MAX_OFFLINE_INSTALL_ATTEMPTS=100
|
||||
fi
|
||||
offline_attempts=1
|
||||
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
|
||||
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
|
||||
crucial_cmd_attempts=1
|
||||
iso_drive_path=""
|
||||
while true; do
|
||||
if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
|
||||
echo "Warning: Offline install timed out!"
|
||||
break
|
||||
fi
|
||||
set +e
|
||||
output=`blkid -o device -t TYPE=iso9660`
|
||||
set -e
|
||||
if [ "$output" != "" ]; then
|
||||
while read -r line; do
|
||||
if [ ! -d "${ISO_MOUNT_DIR}" ]; then
|
||||
mkdir "${ISO_MOUNT_DIR}"
|
||||
fi
|
||||
retval=0
|
||||
set +e
|
||||
mount -o ro "${line}" "${ISO_MOUNT_DIR}"
|
||||
retval=$?
|
||||
set -e
|
||||
if [ $retval -eq 0 ]; then
|
||||
if [ -d "$BINARIES_DIR" ]; then
|
||||
iso_drive_path="${line}"
|
||||
break
|
||||
else
|
||||
umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
|
||||
fi
|
||||
fi
|
||||
done <<< "$output"
|
||||
fi
|
||||
if [ -d "$BINARIES_DIR" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
|
||||
sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
|
||||
offline_attempts=$[$offline_attempts + 1]
|
||||
done
|
||||
|
||||
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
|
||||
export PATH=$PATH:/opt/bin
|
||||
fi
|
||||
|
||||
if [ -d "$BINARIES_DIR" ]; then
|
||||
### Binaries available offline ###
|
||||
echo "Installing binaries from ${BINARIES_DIR}"
|
||||
mkdir -p /opt/bin/
|
||||
tar -zxf ${BINARIES_DIR}/etcd/etcd-linux-amd64.tar.gz -C /opt/bin/
|
||||
mv /opt/bin/etcd*/etcd* /opt/bin/
|
||||
sudo rm -rf /opt/bin/etcd-*
|
||||
fi
|
||||
|
||||
- path: /etc/systemd/system/etcd.service
|
||||
permissions: '0755'
|
||||
owner: root:root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=etcd
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
ExecStart=/opt/bin/etcd \
|
||||
--name {{ etcd.node_name }} \
|
||||
--initial-advertise-peer-urls http://{{ etcd.node_ip }}:2380 \
|
||||
--listen-peer-urls http://{{ etcd.node_ip }}:2380 \
|
||||
--advertise-client-urls http://{{ etcd.node_ip }}:2379 \
|
||||
--listen-client-urls http://{{ etcd.node_ip }}:2379,http://127.0.0.1:2379 \
|
||||
--initial-cluster-token etcd-cluster-1 \
|
||||
--initial-cluster {{ etcd.initial_cluster_nodes }} \
|
||||
--initial-cluster-state new
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
runcmd:
|
||||
- chown -R cloud:cloud /home/cloud/.ssh
|
||||
- /opt/bin/setup-etcd-node
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable --now etcd
|
||||
@ -225,6 +225,33 @@ write_files:
|
||||
done
|
||||
fi
|
||||
|
||||
- path: /etc/kubernetes/kubeadm-config.yaml
|
||||
permissions: '0644'
|
||||
owner: root:root
|
||||
content: |
|
||||
apiVersion: kubeadm.k8s.io/v1beta3
|
||||
kind: ClusterConfiguration
|
||||
apiServer:
|
||||
certSANs:
|
||||
{{ k8s_control.server_ips }}
|
||||
controlPlaneEndpoint: {{ k8s_control.server_ip }}:{{ k8s.api_server_port }}
|
||||
etcd:
|
||||
external:
|
||||
endpoints:
|
||||
{{ etcd.etcd_endpoint_list }}
|
||||
---
|
||||
apiVersion: kubeadm.k8s.io/v1beta3
|
||||
kind: InitConfiguration
|
||||
bootstrapTokens:
|
||||
- token: "{{ k8s_control_node.cluster.token }}"
|
||||
nodeRegistration:
|
||||
criSocket: /run/containerd/containerd.sock
|
||||
localAPIEndpoint:
|
||||
advertiseAddress: {{ k8s_control.server_ip }}
|
||||
bindPort: {{ k8s.api_server_port }}
|
||||
certificateKey: {{ k8s_control.certificate_key }}
|
||||
|
||||
|
||||
- path: /opt/bin/deploy-kube-system
|
||||
permissions: '0700'
|
||||
owner: root:root
|
||||
@ -240,6 +267,7 @@ write_files:
|
||||
export PATH=$PATH:/opt/bin
|
||||
fi
|
||||
|
||||
EXTERNAL_ETCD_NODES={{ etcd.unstacked_etcd }}
|
||||
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
|
||||
crucial_cmd_attempts=1
|
||||
while true; do
|
||||
@ -249,7 +277,11 @@ write_files:
|
||||
fi
|
||||
retval=0
|
||||
set +e
|
||||
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
|
||||
if [[ ${EXTERNAL_ETCD_NODES} == true ]]; then
|
||||
kubeadm init --config /etc/kubernetes/kubeadm-config.yaml --upload-certs
|
||||
else
|
||||
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
|
||||
fi
|
||||
retval=$?
|
||||
set -e
|
||||
if [ $retval -eq 0 ]; then
|
||||
|
||||
@ -19,8 +19,8 @@
|
||||
set -e
|
||||
|
||||
if [ $# -lt 6 ]; then
|
||||
echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME"
|
||||
echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4"
|
||||
echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG ETCD_VER BUILD_NAME"
|
||||
echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml 3.5.1 setup-v1.11.4"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -31,7 +31,7 @@ start_dir="$PWD"
|
||||
iso_dir="/tmp/iso"
|
||||
working_dir="${iso_dir}/"
|
||||
mkdir -p "${working_dir}"
|
||||
build_name="${7}.iso"
|
||||
build_name="${8}.iso"
|
||||
[ -z "${build_name}" ] && build_name="setup-${RELEASE}.iso"
|
||||
|
||||
CNI_VERSION="v${3}"
|
||||
@ -148,6 +148,12 @@ chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm"
|
||||
echo "Updating imagePullPolicy to IfNotPresent in yaml files..."
|
||||
sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml
|
||||
|
||||
# Install etcd dependencies
|
||||
etcd_dir="${working_dir}/etcd"
|
||||
mkdir -p "${etcd_dir}"
|
||||
ETCD_VER=v${7}
|
||||
wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz
|
||||
|
||||
mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}"
|
||||
|
||||
rm -rf "${iso_dir}"
|
||||
|
||||
@ -2443,6 +2443,7 @@
|
||||
"label.quotagb": "Quota in GB",
|
||||
"label.encryption": "Encryption",
|
||||
"label.etcdnodes": "Number of etcd nodes",
|
||||
"label.etcd.ips": "etcd Node(s) IP address(es)",
|
||||
"label.versioning": "Versioning",
|
||||
"label.objectlocking": "Object Lock",
|
||||
"label.bucket.policy": "Bucket Policy",
|
||||
|
||||
@ -188,6 +188,24 @@
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="resource-detail-item" v-if="resource.etcdips">
|
||||
<div class="resource-detail-item__label">{{ $t('label.etcd.ips') }}</div>
|
||||
<div class="resource-detail-item__details resource-detail-item__details--start">
|
||||
<div>
|
||||
<div
|
||||
v-for="(address, index) in resource.etcdips"
|
||||
:key="index">
|
||||
<environment-outlined
|
||||
@click="$message.success(`${$t('label.copied.clipboard')} : ${ address }`)"
|
||||
v-clipboard:copy="address"
|
||||
/>
|
||||
<router-link v-if="address" :to="{ path: '/publicip/' + index }">
|
||||
<copy-label :label="address" />
|
||||
</router-link>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="resource-detail-item" v-if="('cpunumber' in resource && 'cpuspeed' in resource) || resource.cputotal">
|
||||
<div class="resource-detail-item__label">{{ $t('label.cpu') }}</div>
|
||||
<div class="resource-detail-item__details">
|
||||
|
||||
@ -114,7 +114,8 @@
|
||||
<status :text="text ? text : ''" displayText />
|
||||
</template>
|
||||
<template v-if="column.key === 'port'" :name="text" :record="record">
|
||||
{{ cksSshStartingPort + index }}
|
||||
<span v-if="record.isexternalnode || (!record.isexternalnode && !record.isetcdnode)"> {{ cksSshStartingPort + index }} </span>
|
||||
<span v-else> {{ etcdSshPort }} </span>
|
||||
</template>
|
||||
<template v-if="column.key === 'actions'">
|
||||
<a-tooltip placement="bottom" >
|
||||
@ -209,6 +210,7 @@ export default {
|
||||
publicIpAddress: null,
|
||||
currentTab: 'details',
|
||||
cksSshStartingPort: 2222,
|
||||
etcdSshPort: 50000,
|
||||
annotations: []
|
||||
}
|
||||
},
|
||||
@ -280,6 +282,7 @@ export default {
|
||||
dataIndex: 'actions'
|
||||
})
|
||||
}
|
||||
this.fetchEtcdSshPort()
|
||||
this.handleFetchData()
|
||||
this.setCurrentTab()
|
||||
},
|
||||
@ -382,9 +385,10 @@ export default {
|
||||
},
|
||||
fetchInstances () {
|
||||
this.instanceLoading = true
|
||||
var defaultNodes = this.resource.virtualmachines.filter(x => !x.isexternalnode)
|
||||
var defaultNodes = this.resource.virtualmachines.filter(x => !x.isexternalnode && !x.isetcdnode)
|
||||
var externalNodes = this.resource.virtualmachines.filter(x => x.isexternalnode)
|
||||
this.virtualmachines = defaultNodes.concat(externalNodes)
|
||||
var etcdNodes = this.resource.virtualmachines.filter(x => x.isetcdnode)
|
||||
this.virtualmachines = defaultNodes.concat(externalNodes).concat(etcdNodes)
|
||||
this.virtualmachines.map(x => { x.ipaddress = x.nic[0].ipaddress })
|
||||
this.instanceLoading = false
|
||||
},
|
||||
@ -435,6 +439,15 @@ export default {
|
||||
this.networkLoading = false
|
||||
})
|
||||
},
|
||||
fetchEtcdSshPort () {
|
||||
const params = {}
|
||||
params.name = 'cloud.kubernetes.etcd.node.start.port'
|
||||
var apiName = 'listConfigurations'
|
||||
api(apiName, params).then(json => {
|
||||
const configResponse = json.listconfigurationsresponse.configuration
|
||||
this.etcdSshPort = configResponse[0]?.value
|
||||
})
|
||||
},
|
||||
downloadKubernetesClusterConfig () {
|
||||
var blob = new Blob([this.clusterConfig], { type: 'text/plain' })
|
||||
var filename = 'kube.conf'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user