mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
Externalize the CKS Configuration, so that end users can tweak the configuration before deploying the cluster
* Add new directory to c8 packaging for CKS config * Remove k8s configuration from resources and make it configurable * Revert "Remove k8s configuration from resources and make it configurable" This reverts commit d5997033ebe4ba559e6478a64578b894f8e7d3db. * copy conf to mgmt server and consume them from there
This commit is contained in:
Pearl Dsilva
parent
ac53e4b322
commit
249d7106df
8
debian/rules
vendored
8
debian/rules
vendored
@ -70,6 +70,7 @@ override_dh_auto_install:
|
||||
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib
|
||||
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup
|
||||
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm
|
||||
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf
|
||||
mkdir $(DESTDIR)/var/log/$(PACKAGE)/management
|
||||
mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management
|
||||
mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator
|
||||
@ -83,6 +84,7 @@ override_dh_auto_install:
|
||||
cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar
|
||||
cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/
|
||||
cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/
|
||||
cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/
|
||||
rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt
|
||||
|
||||
# Bundle cmk in cloudstack-management
|
||||
@ -95,6 +97,12 @@ override_dh_auto_install:
|
||||
chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
|
||||
|
||||
install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses
|
||||
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml
|
||||
|
||||
# Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF
|
||||
# This should all be in /etc/cloudstack/management
|
||||
ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf
|
||||
|
||||
@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d
|
||||
@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL
|
||||
chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk
|
||||
|
||||
cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
|
||||
|
||||
cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
|
||||
cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
|
||||
cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp
|
||||
cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/
|
||||
@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
|
||||
#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
|
||||
install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management
|
||||
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml
|
||||
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml
|
||||
|
||||
# SystemVM template
|
||||
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
|
||||
cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
|
||||
@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
|
||||
%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm
|
||||
%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption
|
||||
%attr(0755,root,root) %{_bindir}/cmk
|
||||
%{_datadir}/%{name}-management/cks/conf/*.yml
|
||||
%{_datadir}/%{name}-management/setup/*.sql
|
||||
%{_datadir}/%{name}-management/setup/*.sh
|
||||
%{_datadir}/%{name}-management/setup/server-setup.xml
|
||||
|
||||
@ -21,6 +21,9 @@ import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
@ -143,6 +146,7 @@ public class KubernetesClusterActionWorker {
|
||||
|
||||
public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
|
||||
public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes";
|
||||
public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks";
|
||||
|
||||
protected Logger logger = LogManager.getLogger(getClass());
|
||||
|
||||
@ -264,6 +268,11 @@ public class KubernetesClusterActionWorker {
|
||||
return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset());
|
||||
}
|
||||
|
||||
protected String readK8sConfigFile(String resource) throws IOException {
|
||||
Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource));
|
||||
return Files.readString(path);
|
||||
}
|
||||
|
||||
protected String getControlNodeLoginUser() {
|
||||
List<KubernetesClusterVmMapVO> vmMapVOList = getKubernetesClusterVMMaps();
|
||||
if (!vmMapVOList.isEmpty()) {
|
||||
@ -316,7 +325,7 @@ public class KubernetesClusterActionWorker {
|
||||
}
|
||||
|
||||
protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster,
|
||||
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
|
||||
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
|
||||
logMessage(logLevel, message, e);
|
||||
stateTransitTo(kubernetesCluster.getId(), event);
|
||||
detachIsoKubernetesVMs(clusterVMs);
|
||||
@ -670,14 +679,14 @@ public class KubernetesClusterActionWorker {
|
||||
|
||||
try {
|
||||
String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'",
|
||||
scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
|
||||
scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
|
||||
Account account = accountDao.findById(kubernetesCluster.getAccountId());
|
||||
if (account != null && account.getType() == Account.Type.PROJECT) {
|
||||
String projectId = projectService.findByProjectAccountId(account.getId()).getUuid();
|
||||
command = String.format("%s -p '%s'", command, projectId);
|
||||
}
|
||||
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
return result.first();
|
||||
} catch (Exception e) {
|
||||
String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
|
||||
@ -696,7 +705,7 @@ public class KubernetesClusterActionWorker {
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s",
|
||||
kubernetesCluster.getName(), filename), e);
|
||||
kubernetesCluster.getName(), filename), e);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
@ -719,11 +728,11 @@ public class KubernetesClusterActionWorker {
|
||||
sshKeyFile = getManagementServerSshPublicKeyFile();
|
||||
}
|
||||
SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
|
||||
"~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
|
||||
"~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
|
||||
// Ensure destination dir scriptPath exists and copy file to destination
|
||||
String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination);
|
||||
SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
|
||||
cmdStr, 10000, 10000, 10 * 60 * 1000);
|
||||
cmdStr, 10000, 10000, 10 * 60 * 1000);
|
||||
} catch (Exception e) {
|
||||
throw new CloudRuntimeException(e);
|
||||
}
|
||||
@ -771,7 +780,7 @@ public class KubernetesClusterActionWorker {
|
||||
// Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
|
||||
if (manager.isDirectAccess(network)) {
|
||||
logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
|
||||
kubernetesCluster.getName()), null);
|
||||
kubernetesCluster.getName()), null);
|
||||
return true;
|
||||
}
|
||||
File pkFile = getManagementServerSshPublicKeyFile();
|
||||
@ -782,7 +791,7 @@ public class KubernetesClusterActionWorker {
|
||||
try {
|
||||
String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename);
|
||||
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
|
||||
// Maybe the file isn't present. Try and copy it
|
||||
if (!result.first()) {
|
||||
@ -792,12 +801,12 @@ public class KubernetesClusterActionWorker {
|
||||
|
||||
if (!createCloudStackSecret(keys)) {
|
||||
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
|
||||
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
|
||||
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
|
||||
}
|
||||
|
||||
// If at first you don't succeed ...
|
||||
result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
pkFile, null, command, 10000, 10000, 60000);
|
||||
if (!result.first()) {
|
||||
throw new CloudRuntimeException(result.second());
|
||||
}
|
||||
@ -871,7 +880,7 @@ public class KubernetesClusterActionWorker {
|
||||
}
|
||||
|
||||
public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException {
|
||||
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
|
||||
String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml");
|
||||
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
|
||||
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
|
||||
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
|
||||
|
||||
@ -142,9 +142,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
}
|
||||
|
||||
private Pair<String, String> getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
|
||||
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
|
||||
final boolean ejectIso, final boolean externalCni) throws IOException {
|
||||
String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml");
|
||||
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
|
||||
final boolean ejectIso, final boolean externalCni) throws IOException {
|
||||
String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml");
|
||||
final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
|
||||
final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
|
||||
final String caCert = "{{ k8s_control_node.ca.crt }}";
|
||||
@ -170,7 +170,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
|
||||
boolean externalEtcd = !etcdIps.isEmpty();
|
||||
final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes",
|
||||
"kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"),
|
||||
"kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"),
|
||||
addresses, 3650, null);
|
||||
final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate());
|
||||
final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey());
|
||||
@ -301,7 +301,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
}
|
||||
|
||||
private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
|
||||
String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml");
|
||||
String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node-add.yml");
|
||||
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
|
||||
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
|
||||
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
|
||||
@ -336,13 +336,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
private String getInitialEtcdClusterDetails(List<String> ipAddresses, List<String> hostnames) {
|
||||
String initialCluster = "%s=http://%s:%s";
|
||||
StringBuilder clusterInfo = new StringBuilder();
|
||||
for (int i = 0; i < ipAddresses.size(); i++) {
|
||||
clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT));
|
||||
if (i < ipAddresses.size()-1) {
|
||||
clusterInfo.append(",");
|
||||
}
|
||||
for (int i = 0; i < ipAddresses.size(); i++) {
|
||||
clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT));
|
||||
if (i < ipAddresses.size()-1) {
|
||||
clusterInfo.append(",");
|
||||
}
|
||||
return clusterInfo.toString();
|
||||
}
|
||||
return clusterInfo.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -373,7 +373,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
|
||||
private String getEtcdNodeConfig(final List<String> ipAddresses, final List<String> hostnames, final int etcdNodeIndex,
|
||||
final boolean ejectIso) throws IOException {
|
||||
String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml");
|
||||
String k8sEtcdNodeConfig = readK8sConfigFile("/conf/etcd-node.yml");
|
||||
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
|
||||
final String ejectIsoKey = "{{ k8s.eject.iso }}";
|
||||
final String installWaitTime = "{{ k8s.install.wait.time }}";
|
||||
@ -576,7 +576,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
|
||||
}
|
||||
|
||||
private List<Network.IpAddresses> getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) {
|
||||
List<Network.IpAddresses> guestIps = new ArrayList<>();
|
||||
List<Network.IpAddresses> guestIps = new ArrayList<>();
|
||||
for (int i = 1; i <= etcdNodeCount; i++) {
|
||||
guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null));
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user