From 9288c64e5f618e45844f19a902f022c9c9dec8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Weing=C3=A4rtner?= Date: Thu, 19 Apr 2018 15:57:45 -0300 Subject: [PATCH 1/2] systemvm: Use double quotes with 'RROUTER' variable in "common.sh" script (#2586) While debugging the VR for #2579, I noticed that one of the scripts were breaking. The variable RROUTER was not set and this broke a conditional. Signed-off-by: Rohit Yadav --- systemvm/debian/opt/cloud/bin/setup/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index e24a27790b7..d2a26c988b0 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -585,7 +585,7 @@ routing_svcs() { systemctl enable haproxy echo "haproxy apache2" > /var/cache/cloud/enabled_svcs echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs - if [ $RROUTER -eq 1 ] + if [ "$RROUTER" -eq "1" ] then systemctl disable --now dnsmasq systemctl enable conntrackd From 8da2462469db9656e9749ed5607635ce53e8581e Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 20 Apr 2018 00:36:18 +0530 Subject: [PATCH 2/2] CLOUDSTACK-10333: Secure Live VM Migration for KVM (#2505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This extends securing of KVM hosts to securing of libvirt on KVM host as well for TLS enabled live VM migration. To simplify implementation securing of host implies that both host and libvirtd processes are secured with management server's CA plugin issued certificates. Based on whether keystore and certificates files are available at /etc/cloudstack/agent, the KVM agent determines whether to use TLS or TCP based uris for live VM migration. It is also enforced that a secured host will allow live VM migration to/from other secured host, and an unsecured hosts will allow live VM migration to/from other unsecured host only. Post upgrade the KVM agent on startup will expose its security state (secured detail is sent as true or false) to the managements server that gets saved in host_details for the host. This host detail can be accesed via the listHosts response, and in the UI unsecured KVM hosts will show up with the host state of ‘unsecured’. Further, a button has been added that allows admins to provision/renew certificates to KVM hosts and can be used to secure any unsecured KVM host. The `cloudstack-setup-agent` was modified to accept a new flag `-s` which will reconfigure libvirtd with following settings: listen_tcp=0 listen_tls=1 tcp_port="16509" tls_port="16514" auth_tcp="none" auth_tls="none" key_file = "/etc/pki/libvirt/private/serverkey.pem" cert_file = "/etc/pki/libvirt/servercert.pem" ca_file = "/etc/pki/CA/cacert.pem" For a connected KVM host agent, when the certificate are renewed/provisioned a background task is scheduled that waits until all of the agent tasks finish after which libvirt process is restarted and finally the agent is restarted via AgentShell. There are no API or DB changes. Signed-off-by: Rohit Yadav --- agent/bindir/cloud-setup-agent.in | 12 +- agent/src/com/cloud/agent/Agent.java | 155 ++++++++- agent/src/com/cloud/agent/AgentShell.java | 7 +- agent/src/com/cloud/agent/IAgentShell.java | 5 + .../VirtualRoutingResource.java | 18 +- .../ca/PostCertificateRenewalCommand.java | 34 ++ .../ca/SetupCertificateCommand.java | 6 +- debian/cloudstack-agent.postinst | 1 + .../ca/provider/RootCAProvider.java | 2 +- .../resource/LibvirtComputingResource.java | 43 ++- .../wrapper/LibvirtMigrateCommandWrapper.java | 20 +- .../LibvirtModifyTargetsCommandWrapper.java | 2 +- ...tPostCertificateRenewalCommandWrapper.java | 52 +++ .../LibvirtMigrateCommandWrapperTest.java | 22 ++ python/lib/cloud_utils.py | 4 +- python/lib/cloudutils/serviceConfig.py | 35 +- scripts/util/keystore-cert-import | 13 + .../discoverer/LibvirtServerDiscoverer.java | 32 +- .../apache/cloudstack/ca/CAManagerImpl.java | 3 +- test/integration/smoke/test_vm_life_cycle.py | 310 +++++++++++++++++- ui/css/cloudstack3.css | 2 + ui/l10n/en.js | 2 + ui/scripts/system.js | 51 ++- .../main/java/com/cloud/utils/nio/Link.java | 8 +- .../java/com/cloud/utils/script/Script.java | 2 +- .../com/cloud/utils/ssh/SSHCmdHelper.java | 4 +- .../utils/security/KeyStoreUtils.java | 39 ++- 27 files changed, 775 insertions(+), 109 deletions(-) create mode 100644 core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java create mode 100644 plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 8d2b91961ae..3c6203c2d34 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -26,6 +26,7 @@ from cloudutils.configFileOps import configFileOps from cloudutils.globalEnv import globalEnv from cloudutils.networkConfig import networkConfig from cloudutils.syscfg import sysConfigFactory +from cloudutils.serviceConfig import configureLibvirtConfig from optparse import OptionParser @@ -100,6 +101,7 @@ if __name__ == '__main__': parser.add_option("-c", "--cluster", dest="cluster", help="cluster id") parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type") parser.add_option("-g", "--guid", dest="guid", help="guid") + parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd") parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface") parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface") parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface") @@ -110,6 +112,12 @@ if __name__ == '__main__': glbEnv.bridgeType = bridgeType (options, args) = parser.parse_args() + + if not options.auto and options.secure: + configureLibvirtConfig(True) + print "Libvirtd with TLS configured" + sys.exit(0) + if options.auto is None: userInputs = getUserInputs() glbEnv.mgtSvr = userInputs[0] @@ -138,7 +146,9 @@ if __name__ == '__main__': glbEnv.nics.append(options.prvNic) glbEnv.nics.append(options.pubNic) glbEnv.nics.append(options.guestNic) - + + glbEnv.secure = options.secure + print "Starting to configure your system:" syscfg = sysConfigFactory.getSysConfigFactory(glbEnv) try: diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index 32112540c1c..90e37909434 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -42,6 +42,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; @@ -68,6 +69,7 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentControlChannelException; +import com.cloud.host.Host; import com.cloud.resource.ServerResource; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; @@ -127,6 +129,7 @@ public class Agent implements HandlerFactory, IAgentControl { Long _id; Timer _timer = new Timer("Agent Timer"); + Timer certTimer; Timer hostLBTimer; List _watchList = new ArrayList(); @@ -140,9 +143,11 @@ public class Agent implements HandlerFactory, IAgentControl { long _startupWait = _startupWaitDefault; boolean _reconnectAllowed = true; //For time sentitive task, e.g. PingTask - private final ThreadPoolExecutor _ugentTaskPool; + ThreadPoolExecutor _ugentTaskPool; ExecutorService _executor; + Thread _shutdownThread = new ShutdownThread(this); + private String _keystoreSetupPath; private String _keystoreCertImportPath; @@ -153,7 +158,7 @@ public class Agent implements HandlerFactory, IAgentControl { _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -192,7 +197,7 @@ public class Agent implements HandlerFactory, IAgentControl { // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp()); s_logger.debug("Adding shutdown hook"); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -239,20 +244,39 @@ public class Agent implements HandlerFactory, IAgentControl { return _resource.getClass().getSimpleName(); } + /** + * In case of a software based agent restart, this method + * can help to perform explicit garbage collection of any old + * agent instances and its inner objects. + */ + private void scavengeOldAgentObjects() { + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); + } + public void start() { if (!_resource.start()) { s_logger.error("Unable to start the resource: " + _resource.getName()); throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName()); } - _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript); + _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT); if (_keystoreSetupPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT)); } - _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript); + _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT); if (_keystoreCertImportPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT)); } try { @@ -274,6 +298,7 @@ public class Agent implements HandlerFactory, IAgentControl { } } _shell.updateConnectedHost(); + scavengeOldAgentObjects(); } public void stop(final String reason, final String detail) { @@ -298,6 +323,7 @@ public class Agent implements HandlerFactory, IAgentControl { } _connection.stop(); _connection = null; + _link = null; } if (_resource != null) { @@ -305,7 +331,34 @@ public class Agent implements HandlerFactory, IAgentControl { _resource = null; } - _ugentTaskPool.shutdownNow(); + if (_startup != null) { + _startup = null; + } + + if (_ugentTaskPool != null) { + _ugentTaskPool.shutdownNow(); + _ugentTaskPool = null; + } + + if (_executor != null) { + _executor.shutdown(); + _executor = null; + } + + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + + if (hostLBTimer != null) { + hostLBTimer.cancel(); + hostLBTimer = null; + } + + if (certTimer != null) { + certTimer.cancel(); + certTimer = null; + } } public Long getId() { @@ -318,6 +371,15 @@ public class Agent implements HandlerFactory, IAgentControl { _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id)); } + private synchronized void scheduleServicesRestartTask() { + if (certTimer != null) { + certTimer.cancel(); + certTimer.purge(); + } + certTimer = new Timer("Certificate Renewal Timer"); + certTimer.schedule(new PostCertificateRenewalTask(this), 5000L); + } + private synchronized void scheduleHostLBCheckerTask(final long checkInterval) { if (hostLBTimer != null) { hostLBTimer.cancel(); @@ -578,6 +640,9 @@ public class Agent implements HandlerFactory, IAgentControl { answer = setupAgentKeystore((SetupKeyStoreCommand) cmd); } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) { answer = setupAgentCertificate((SetupCertificateCommand) cmd); + if (Host.Type.Routing.equals(_resource.getType())) { + scheduleServicesRestartTask(); + } } else if (cmd instanceof SetupDirectDownloadCertificate) { answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd); } else if (cmd instanceof SetupMSListCommand) { @@ -641,7 +706,7 @@ public class Agent implements HandlerFactory, IAgentControl { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; String cerFile = agentFile.getParent() + "/" + certificateName + ".cer"; Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile)); @@ -666,13 +731,13 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCsrFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.CSR_FILENAME; - String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName); + String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (Strings.isNullOrEmpty(storedPassword)) { storedPassword = keyStorePassword; - _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword); + _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword); } Script script = new Script(true, _keystoreSetupPath, 60000, s_logger); @@ -706,10 +771,10 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCertFile; - final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultPrivateKeyFile; - final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCaCertFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME; + final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.PKEY_FILENAME; + final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.CACERT_FILENAME; try { FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset()); @@ -722,7 +787,7 @@ public class Agent implements HandlerFactory, IAgentControl { Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger); script.add(agentFile.getAbsolutePath()); script.add(keyStoreFile); - script.add(KeyStoreUtils.agentMode); + script.add(KeyStoreUtils.AGENT_MODE); script.add(certFile); script.add(""); script.add(caCertFile); @@ -1072,6 +1137,60 @@ public class Agent implements HandlerFactory, IAgentControl { } } + /** + * Task stops the current agent and launches a new agent + * when there are no outstanding jobs in the agent's task queue + */ + public class PostCertificateRenewalTask extends ManagedContextTimerTask { + + private Agent agent; + + public PostCertificateRenewalTask(final Agent agent) { + this.agent = agent; + } + + @Override + protected void runInContext() { + while (true) { + try { + if (_inProgress.get() == 0) { + s_logger.debug("Running post certificate renewal task to restart services."); + + // Let the resource perform any post certificate renewal cleanups + _resource.executeRequest(new PostCertificateRenewalCommand()); + + IAgentShell shell = agent._shell; + ServerResource resource = agent._resource.getClass().newInstance(); + + // Stop current agent + agent.cancelTasks(); + agent._reconnectAllowed = false; + Runtime.getRuntime().removeShutdownHook(agent._shutdownThread); + agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates"); + + // Nullify references for GC + agent._shell = null; + agent._watchList = null; + agent._shutdownThread = null; + agent._controlListeners = null; + agent = null; + + // Start a new agent instance + shell.launchNewAgent(resource); + return; + } + if (s_logger.isTraceEnabled()) { + s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds"); + } + Thread.sleep(5000); + } catch (final Exception e) { + s_logger.warn("Failed to execute post certificate renewal command:", e); + break; + } + } + } + } + public class PreferredHostCheckerTask extends ManagedContextTimerTask { @Override diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index 13b6c65a351..01654ac9caa 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -419,7 +419,7 @@ public class AgentShell implements IAgentShell, Daemon { final Constructor constructor = impl.getDeclaredConstructor(); constructor.setAccessible(true); ServerResource resource = (ServerResource)constructor.newInstance(); - launchAgent(getNextAgentId(), resource); + launchNewAgent(resource); } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { @@ -447,9 +447,10 @@ public class AgentShell implements IAgentShell, Daemon { s_logger.trace("Launching agent based on type=" + typeInfo); } - private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException { + public void launchNewAgent(ServerResource resource) throws ConfigurationException { // we don't track agent after it is launched for now - Agent agent = new Agent(this, localAgentId, resource); + _agents.clear(); + Agent agent = new Agent(this, getNextAgentId(), resource); _agents.add(agent); agent.start(); } diff --git a/agent/src/com/cloud/agent/IAgentShell.java b/agent/src/com/cloud/agent/IAgentShell.java index 5b52cee6361..5d389a07041 100644 --- a/agent/src/com/cloud/agent/IAgentShell.java +++ b/agent/src/com/cloud/agent/IAgentShell.java @@ -19,6 +19,9 @@ package com.cloud.agent; import java.util.Map; import java.util.Properties; +import javax.naming.ConfigurationException; + +import com.cloud.resource.ServerResource; import com.cloud.utils.backoff.BackoffAlgorithm; public interface IAgentShell { @@ -66,4 +69,6 @@ public interface IAgentShell { void updateConnectedHost(); String getConnectedHost(); + + void launchNewAgent(ServerResource resource) throws ConfigurationException; } diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index d7c3d56f9b9..0ffe8cc0ea2 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -157,11 +157,11 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s " + "%s %d " + "/usr/local/cloud/systemvm/conf/%s", - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_FILENAME, cmd.getKeystorePassword(), cmd.getValidityDays(), - KeyStoreUtils.defaultCsrFile); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args); + KeyStoreUtils.CSR_FILENAME); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args); return new SetupKeystoreAnswer(result.getDetails()); } @@ -171,15 +171,15 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\"", - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, cmd.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, cmd.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, cmd.getEncodedPrivateKey()); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args); return new SetupCertificateAnswer(result.isSuccess()); } diff --git a/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java new file mode 100644 index 00000000000..12df6196128 --- /dev/null +++ b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.ca; + +import com.cloud.agent.api.Command; + +public class PostCertificateRenewalCommand extends Command { + + public PostCertificateRenewalCommand() { + super(); + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java index 1cd31509d39..7727282bcee 100644 --- a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java +++ b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java @@ -82,15 +82,15 @@ public class SetupCertificateCommand extends NetworkElementCommand { } public String getEncodedPrivateKey() { - return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCertificate() { - return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCaCertificates() { - return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public boolean isHandleByAgent() { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index a9b8b687fab..c358c3ca680 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -50,6 +50,7 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu + ;; esac diff --git a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java index 4a3585ac4e2..6584b35861a 100644 --- a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -241,7 +241,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con @Override public char[] getKeyStorePassphrase() { - return KeyStoreUtils.defaultKeystorePassphrase; + return KeyStoreUtils.DEFAULT_KS_PASSPHRASE; } ///////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dd039e54263..fc5e5395b87 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -41,11 +41,19 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; +import org.apache.cloudstack.utils.linux.CPUStat; +import org.apache.cloudstack.utils.linux.MemStat; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; @@ -68,14 +76,6 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import com.google.common.base.Strings; - -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; -import org.apache.cloudstack.utils.linux.CPUStat; -import org.apache.cloudstack.utils.linux.MemStat; -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -168,6 +168,7 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; /** * LibvirtComputingResource execute requests on the computing/routing host using @@ -239,6 +240,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected long _hypervisorLibvirtVersion; protected long _hypervisorQemuVersion; protected String _hypervisorPath; + protected String _hostDistro; protected String _networkDirectSourceMode; protected String _networkDirectDevice; protected String _sysvmISOPath; @@ -2599,11 +2601,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); cmd.setIqn(getIqn()); + if (cmd.getHostDetails().containsKey("Host.OS")) { + _hostDistro = cmd.getHostDetails().get("Host.OS"); + } + StartupStorageCommand sscmd = null; try { @@ -3777,4 +3784,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public long getTotalMemory() { return _totalMemory; } + + public String getHostDistro() { + return _hostDistro; + } + + public boolean isHostSecured() { + // Test for host certificates + final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); + if (confFile == null || !confFile.exists() || !new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists()) { + return false; + } + + // Test for libvirt TLS configuration + try { + new Connect(String.format("qemu+tls://%s/system", _privateIp)); + } catch (final LibvirtException ignored) { + return false; + } + return true; + } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index ad32759a517..67ec1b731af 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -20,8 +20,8 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,12 +46,10 @@ import javax.xml.transform.stream.StreamResult; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; - import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo.DomainState; import org.libvirt.LibvirtException; - import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; @@ -71,6 +69,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Strings; @ResourceWrapper(handles = MigrateCommand.class) public final class LibvirtMigrateCommandWrapper extends CommandWrapper { @@ -80,9 +79,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, command.isAutoConvergence(), vmName, command.getDestinationIp()); @@ -203,6 +210,9 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper { - private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class); + private static final Logger s_logger = Logger.getLogger(LibvirtModifyTargetsCommandWrapper.class); @Override public Answer execute(final ModifyTargetsCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java new file mode 100644 index 00000000000..df89d2470dd --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = PostCertificateRenewalCommand.class) +public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtPostCertificateRenewalCommandWrapper.class); + + @Override + public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) { + s_logger.info("Restarting libvirt after certificate provisioning/renewal"); + if (command != null) { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, s_logger); + if ("Ubuntu".equals(serverResource.getHostDistro()) || "Debian".equals(serverResource.getHostDistro())) { + script.add("libvirt-bin"); + } else { + script.add("libvirtd"); + } + script.add("restart"); + script.execute(); + return new SetupCertificateAnswer(true); + } + return new SetupCertificateAnswer(false); + } +} diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index ed13cb25f0c..da71e40c30f 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -18,10 +18,14 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.junit.Test; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.exception.CloudRuntimeException; + public class LibvirtMigrateCommandWrapperTest { String fullfile = "\n" + @@ -303,4 +307,22 @@ public class LibvirtMigrateCommandWrapperTest { final String result = lw.replaceIpForVNCInDescFile(xmlDesc, targetIp); assertTrue("transformation does not live up to expectation:\n" + result, expectedXmlDesc.equals(result)); } + + @Test + public void testMigrationUri() { + final String ip = "10.1.1.1"; + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + LibvirtComputingResource lcr = new LibvirtComputingResource(); + if (lcr.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tls://%s/system", ip)); + } else { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tcp://%s/system", ip)); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testMigrationUriException() { + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + lw.createMigrationURI(null, new LibvirtComputingResource()); + } } diff --git a/python/lib/cloud_utils.py b/python/lib/cloud_utils.py index a9afb04dd8e..36ce617b3f1 100644 --- a/python/lib/cloud_utils.py +++ b/python/lib/cloud_utils.py @@ -802,9 +802,9 @@ class SetupFirewall(ConfigTask): rule = "-p tcp -m tcp --dport 16509 -j ACCEPT" if rule in iptablessave().stdout: return True return False - + def execute(self): - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() if distro in (Fedora , CentOS, RHEL6): for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT') o = service.iptables.save() ; print o.stdout + o.stderr diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index 68d1b9cef5b..2b27868db05 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -471,6 +471,23 @@ class securityPolicyConfigRedhat(serviceCfgBase): logging.debug(formatExceptionInfo()) return False +def configureLibvirtConfig(tls_enabled = True, cfg = None): + cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg) + if tls_enabled: + cfo.addEntry("listen_tcp", "0") + cfo.addEntry("listen_tls", "1") + cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"") + cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"") + cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"") + else: + cfo.addEntry("listen_tcp", "1") + cfo.addEntry("listen_tls", "0") + cfo.addEntry("tcp_port", "\"16509\"") + cfo.addEntry("tls_port", "\"16514\"") + cfo.addEntry("auth_tcp", "\"none\"") + cfo.addEntry("auth_tls", "\"none\"") + cfo.save() + class libvirtConfigRedhat(serviceCfgBase): def __init__(self, syscfg): super(libvirtConfigRedhat, self).__init__(syscfg) @@ -478,12 +495,7 @@ class libvirtConfigRedhat(serviceCfgBase): def config(self): try: - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\"") - cfo.addEntry("auth_tcp", "\"none\"") - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) cfo = configFileOps("/etc/sysconfig/libvirtd", self) cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'") @@ -516,12 +528,7 @@ class libvirtConfigUbuntu(serviceCfgBase): self.serviceName = "Libvirt" def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) if os.path.exists("/etc/init/libvirt-bin.conf"): cfo = configFileOps("/etc/init/libvirt-bin.conf", self) @@ -567,7 +574,7 @@ class firewallConfigUbuntu(serviceCfgBase): def config(self): try: - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() for p in ports: bash("ufw allow %s"%p) bash("ufw allow proto tcp from any to any port 5900:6100") @@ -627,7 +634,7 @@ class firewallConfigBase(serviceCfgBase): class firewallConfigAgent(firewallConfigBase): def __init__(self, syscfg): super(firewallConfigAgent, self).__init__(syscfg) - self.ports = "22 16509 5900:6100 49152:49216".split() + self.ports = "22 16509 16514 5900:6100 49152:49216".split() if syscfg.env.distribution.getVersion() == "CentOS": self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"] else: diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index 67ce3400345..96196d93902 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" +LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -78,6 +79,18 @@ fi rm -f "$NEW_KS_FILE.p12" mv -f "$NEW_KS_FILE" "$KS_FILE" +# Secure libvirtd on cert import +if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/CA + mkdir -p /etc/pki/libvirt/private + ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem + cloudstack-setup-agent -s > /dev/null +fi + # Update ca-certs if we're in systemvm if [ -f "$SYSTEM_FILE" ]; then mkdir -p /usr/local/share/ca-certificates/cloudstack diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index ef69fdcb436..0b8b40b1b9b 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.discoverer; import java.net.InetAddress; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -141,11 +142,6 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements } private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) { - if (!caManager.canProvisionCertificates()) { - s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate"); - return; - } - if (sshConnection == null) { throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp); } @@ -161,17 +157,17 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s " + "%s %d " + "/etc/cloudstack/agent/%s", - KeyStoreUtils.keyStoreSetupScript, - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_SETUP_SCRIPT, + KeyStoreUtils.KS_FILENAME, PasswordGenerator.generateRandomPassword(16), validityPeriod, - KeyStoreUtils.defaultCsrFile)); + KeyStoreUtils.CSR_FILENAME)); if (!keystoreSetupResult.isSuccess()) { throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp); } - final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null); + final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null); if (certificate == null || certificate.getClientCertificate() == null) { throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp); } @@ -184,14 +180,14 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\"", - KeyStoreUtils.keyStoreImportScript, - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_IMPORT_SCRIPT, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, certificateCommand.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, certificateCommand.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, certificateCommand.getEncodedPrivateKey())); if (setupCertResult != null && !setupCertResult.isSuccess()) { @@ -288,9 +284,13 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic; } + if (!caManager.canProvisionCertificates()) { + throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency."); + } + setupAgentSecurity(sshConnection, agentIp, hostname); - String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a"; + String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a -s "; parameters += " --pubNic=" + kvmPublicNic; parameters += " --prvNic=" + kvmPrivateNic; diff --git a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java index 3a904315fe1..23a0379305b 100644 --- a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java @@ -27,7 +27,6 @@ import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -191,7 +190,7 @@ public class CAManagerImpl extends ManagerBase implements CAManager { if (Strings.isNullOrEmpty(csr)) { return false; } - final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); + final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); } catch (final AgentUnavailableException | OperationTimedoutException e) { LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e); diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3504849786b..d882c1fc9ae 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -21,9 +21,11 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.cloudstackAPI import (recoverVirtualMachine, destroyVirtualMachine, attachIso, - detachIso) -from marvin.lib.utils import (cleanup_resources, - validateList) + detachIso, + provisionCertificate, + updateConfiguration) +from marvin.lib.utils import * + from marvin.lib.base import (Account, ServiceOffering, VirtualMachine, @@ -33,11 +35,13 @@ from marvin.lib.base import (Account, Configurations) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_hosts) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr #Import System modules import time +import re _multiprocess_shared_ = True class TestDeployVM(cloudstackTestCase): @@ -781,3 +785,301 @@ class TestVMLifeCycle(cloudstackTestCase): "Check if ISO is detached from virtual machine" ) return + +class TestSecuredVmMigration(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSecuredVmMigration, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"] + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls._cleanup = [ + cls.small_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + + cls.apiclient = super(TestSecuredVmMigration, cls).getClsTestClient().getApiClient() + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + self.updateConfiguration("ca.plugin.root.auth.strictness", "false") + self.make_all_hosts_secure() + + if self.hypervisor.lower() not in ["kvm"]: + self.skipTest("Secured migration is not supported on other than KVM") + + def tearDown(self): + self.make_all_hosts_secure() + + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_secured_vm_migration(self): + """Test secured VM migration""" + + # Validate the following + # 1. Environment has enough hosts for migration + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + + hosts = self.get_hosts() + + secured_hosts = [] + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + if len(secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + + origin_host = secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tls') + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_not_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + #self.skipTest() + # Validate the following + # 1. Prepare 2 hosts to run in non-secured more + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + hosts = self.get_hosts() + for host in hosts: + self.make_unsecure_connection(host) + + non_secured_hosts = [] + + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'false': + non_secured_hosts.append(host) + + if len(non_secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + origin_host = non_secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='false', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tcp') + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_secured_to_nonsecured_vm_migration(self): + """Test destroy Virtual Machine + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to a Secured host + # 3. Migrates the VM to the non-secured host and assers the migration is via TCP. + + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(secured_hosts[0]) + try: + self.migrate_and_check(origin_host=secured_hosts[0], destination_host=non_secured_host, proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_nonsecured_to_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to the non-secured host + # 3. Migrates the VM to the secured host and assers the migration is via TCP. + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + + hosts = self.get_hosts() + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(non_secured_host) + + try: + self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp') + except Exception: + pass + else: + self.fail("Migration succeed, instead it should fail") + + def get_target_host(self, secured, virtualmachineid): + target_hosts = Host.listForMigration(self.apiclient, + virtualmachineid=virtualmachineid) + for host in target_hosts: + h = list_hosts(self.apiclient,type='Routing', id=host.id)[0] + if h.details.secured == secured: + return h + + cloudstackTestCase.skipTest(self, "No target hosts available, skipping test.") + + def check_migration_protocol(self, protocol, host): + resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("grep -a Live /var/log/cloudstack/agent/agent.log | tail -1") + + if protocol not in resp[0]: + cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n" + "Instead we got: " + resp[0]) + + def make_unsecure_connection(self, host): + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("rm -f /etc/cloudstack/agent/cloud*") + + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i 's/listen_tls.*/listen_tls=0/g' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf ") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i '/.*_file.*/d' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service libvirtd restart") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service cloudstack-agent restart") + + self.check_connection(host=host, secured='false') + time.sleep(10) + return host + + def make_all_hosts_secure(self): + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + for host in hosts: + cmd = provisionCertificate.provisionCertificateCmd() + cmd.hostid = host.id + self.apiclient.updateConfiguration(cmd) + + for host in hosts: + self.check_connection(secured='true', host=host) + + def get_hosts(self): + + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed") + return hosts + + def deploy_vm(self, origin_host): + return VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id, + mode=self.services["mode"], + hostid=origin_host.id + ) + + def check_connection(self, secured, host, retries=5, interval=5): + + while retries > -1: + time.sleep(interval) + host = Host.list( + self.apiclient, + zoneid=self.zone.id, + hostid=host.id, + type='Routing' + )[0] + if host.details.secured != secured: + if retries >= 0: + retries = retries - 1 + continue + else: + return + + raise Exception("Host communication is not as expected: " + secured + + ". Instead it's: " + host.details.secured) + + def migrate_and_check(self, origin_host, destination_host, proto): + + self.vm_to_migrate.migrate(self.apiclient, hostid=destination_host.id) + + self.check_migration_protocol(protocol=proto, host=origin_host) + + vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0] + + self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM") + + def updateConfiguration(self, name, value): + cmd = updateConfiguration.updateConfigurationCmd() + cmd.name = name + cmd.value = value + self.apiclient.updateConfiguration(cmd) \ No newline at end of file diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index e7d4a0cfe41..d68b4f8c3cf 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -12612,11 +12612,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it background-position: -101px -647px; } +.secureKVMHost .icon, .resetPassword .icon, .changePassword .icon { background-position: -68px -30px; } +.secureKVMHost:hover .icon, .resetPassword:hover .icon, .changePassword:hover .icon { background-position: -68px -612px; diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 7b19946b1ed..745deb11ca2 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -270,6 +270,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "label.action.restore.instance.processing":"Restoring Instance....", "label.action.revert.snapshot":"Revert to Snapshot", "label.action.revert.snapshot.processing":"Reverting to Snapshot...", +"label.action.secure.host":"Provision Host Security Keys", "label.action.start.instance":"Start Instance", "label.action.start.instance.processing":"Starting Instance....", "label.action.start.router":"Start Router", @@ -1910,6 +1911,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "message.action.reset.password.warning":"Your instance must be stopped before attempting to change its current password.", "message.action.restore.instance":"Please confirm that you want to restore this instance.", "message.action.revert.snapshot":"Please confirm that you want to revert the owning volume to this snapshot.", +"message.action.secure.host":"This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm?", "message.action.start.instance":"Please confirm that you want to start this instance.", "message.action.start.router":"Please confirm that you want to start this router.", "message.action.start.systemvm":"Please confirm that you want to start this system VM.", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 07520df1d46..63f549ad292 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -9200,6 +9200,11 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } + }); } @@ -15713,7 +15718,8 @@ 'Down': 'off', 'Disconnected': 'off', 'Alert': 'off', - 'Error': 'off' + 'Error': 'off', + 'Unsecure': 'warning' } }, powerstate: { @@ -15761,6 +15767,10 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } }); } @@ -16530,6 +16540,40 @@ } }, + secureKVMHost: { + label: 'label.action.secure.host', + action: function(args) { + var data = { + hostid: args.context.hosts[0].id + }; + $.ajax({ + url: createURL('provisionCertificate'), + data: data, + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.provisioncertificateresponse.jobid, + getActionFilter: function () { + return hostActionfilter; + } + } + }); + } + }); + }, + messages: { + confirm: function (args) { + return 'message.action.secure.host'; + }, + notification: function (args) { + return 'label.action.secure.host'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, enableMaintenanceMode: { label: 'label.action.enable.maintenance.mode', @@ -21968,6 +22012,11 @@ if (jsonObj.state != "Disconnected") allowedActions.push("forceReconnect"); + + if (jsonObj.hypervisor == "KVM") { + allowedActions.push("secureKVMHost"); + } + } else if (jsonObj.resourcestate == "ErrorInMaintenance") { allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 25f6662c522..65824408770 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -379,7 +379,7 @@ public class Link { return caService.createSSLEngine(sslContext, clientAddress); } s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine"); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase); final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); @@ -409,11 +409,11 @@ public class Link { } public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException { - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; File confFile = PropertiesUtil.findConfigFile("agent.properties"); if (confFile != null) { s_logger.info("Conf file found: " + confFile.getAbsolutePath()); - final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.passphrasePropertyName); + final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (pass != null) { passphrase = pass.toCharArray(); } @@ -421,7 +421,7 @@ public class Link { InputStream stream = null; if (confFile != null) { - final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; if (new File(keystorePath).exists()) { stream = new FileInputStream(keystorePath); } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 01f18bda2d2..b73371ac715 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -203,7 +203,7 @@ public class Script implements Callable { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]); + _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]); } try { diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 10407b65642..5324cdcbc18 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -139,7 +139,7 @@ public class SSHCmdHelper { } public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException { - s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]); + s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]); Session sshSession = null; try { sshSession = sshConnection.openSession(); @@ -202,7 +202,7 @@ public class SSHCmdHelper { final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString()); if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) { - s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); + s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); } // exit status delivery might get delayed diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java index 8259d77d4ff..c6f8d21918c 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java @@ -19,23 +19,34 @@ package org.apache.cloudstack.utils.security; +import java.io.File; + +import com.cloud.utils.PropertiesUtil; + public class KeyStoreUtils { + public static final String KS_SETUP_SCRIPT = "keystore-setup"; + public static final String KS_IMPORT_SCRIPT = "keystore-cert-import"; - public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks"; - public static String defaultKeystoreFile = "cloud.jks"; - public static String defaultPrivateKeyFile = "cloud.key"; - public static String defaultCsrFile = "cloud.csr"; - public static String defaultCertFile = "cloud.crt"; - public static String defaultCaCertFile = "cloud.ca.crt"; - public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray(); + public static final String AGENT_PROPSFILE = "agent.properties"; + public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase"; - public static String certNewlineEncoder = "^"; - public static String certSpaceEncoder = "~"; + public static final String KS_FILENAME = "cloud.jks"; + public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray(); - public static String keyStoreSetupScript = "keystore-setup"; - public static String keyStoreImportScript = "keystore-cert-import"; - public static String passphrasePropertyName = "keystore.passphrase"; + public static final String CACERT_FILENAME = "cloud.ca.crt"; + public static final String CERT_FILENAME = "cloud.crt"; + public static final String CSR_FILENAME = "cloud.csr"; + public static final String PKEY_FILENAME = "cloud.key"; - public static String sshMode = "ssh"; - public static String agentMode = "agent"; + public static final String CERT_NEWLINE_ENCODER = "^"; + public static final String CERT_SPACE_ENCODER = "~"; + + public static final String SSH_MODE = "ssh"; + public static final String AGENT_MODE = "agent"; + public static final String SECURED = "secured"; + + public static boolean isHostSecured() { + final File confFile = PropertiesUtil.findConfigFile("agent.properties"); + return confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + CERT_FILENAME).exists(); + } }