diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 8d2b91961ae..3c6203c2d34 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -26,6 +26,7 @@ from cloudutils.configFileOps import configFileOps from cloudutils.globalEnv import globalEnv from cloudutils.networkConfig import networkConfig from cloudutils.syscfg import sysConfigFactory +from cloudutils.serviceConfig import configureLibvirtConfig from optparse import OptionParser @@ -100,6 +101,7 @@ if __name__ == '__main__': parser.add_option("-c", "--cluster", dest="cluster", help="cluster id") parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type") parser.add_option("-g", "--guid", dest="guid", help="guid") + parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd") parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface") parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface") parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface") @@ -110,6 +112,12 @@ if __name__ == '__main__': glbEnv.bridgeType = bridgeType (options, args) = parser.parse_args() + + if not options.auto and options.secure: + configureLibvirtConfig(True) + print "Libvirtd with TLS configured" + sys.exit(0) + if options.auto is None: userInputs = getUserInputs() glbEnv.mgtSvr = userInputs[0] @@ -138,7 +146,9 @@ if __name__ == '__main__': glbEnv.nics.append(options.prvNic) glbEnv.nics.append(options.pubNic) glbEnv.nics.append(options.guestNic) - + + glbEnv.secure = options.secure + print "Starting to configure your system:" syscfg = sysConfigFactory.getSysConfigFactory(glbEnv) try: diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 32112540c1c..90e37909434 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -42,6 +42,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; @@ -68,6 +69,7 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentControlChannelException; +import com.cloud.host.Host; import com.cloud.resource.ServerResource; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; @@ -127,6 +129,7 @@ public class Agent implements HandlerFactory, IAgentControl { Long _id; Timer _timer = new Timer("Agent Timer"); + Timer certTimer; Timer hostLBTimer; List _watchList = new ArrayList(); @@ -140,9 +143,11 @@ public class Agent implements HandlerFactory, IAgentControl { long _startupWait = _startupWaitDefault; boolean _reconnectAllowed = true; //For time sentitive task, e.g. PingTask - private final ThreadPoolExecutor _ugentTaskPool; + ThreadPoolExecutor _ugentTaskPool; ExecutorService _executor; + Thread _shutdownThread = new ShutdownThread(this); + private String _keystoreSetupPath; private String _keystoreCertImportPath; @@ -153,7 +158,7 @@ public class Agent implements HandlerFactory, IAgentControl { _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -192,7 +197,7 @@ public class Agent implements HandlerFactory, IAgentControl { // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp()); s_logger.debug("Adding shutdown hook"); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -239,20 +244,39 @@ public class Agent implements HandlerFactory, IAgentControl { return _resource.getClass().getSimpleName(); } + /** + * In case of a software based agent restart, this method + * can help to perform explicit garbage collection of any old + * agent instances and its inner objects. + */ + private void scavengeOldAgentObjects() { + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); + } + public void start() { if (!_resource.start()) { s_logger.error("Unable to start the resource: " + _resource.getName()); throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName()); } - _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript); + _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT); if (_keystoreSetupPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT)); } - _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript); + _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT); if (_keystoreCertImportPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT)); } try { @@ -274,6 +298,7 @@ public class Agent implements HandlerFactory, IAgentControl { } } _shell.updateConnectedHost(); + scavengeOldAgentObjects(); } public void stop(final String reason, final String detail) { @@ -298,6 +323,7 @@ public class Agent implements HandlerFactory, IAgentControl { } _connection.stop(); _connection = null; + _link = null; } if (_resource != null) { @@ -305,7 +331,34 @@ public class Agent implements HandlerFactory, IAgentControl { _resource = null; } - _ugentTaskPool.shutdownNow(); + if (_startup != null) { + _startup = null; + } + + if (_ugentTaskPool != null) { + _ugentTaskPool.shutdownNow(); + _ugentTaskPool = null; + } + + if (_executor != null) { + _executor.shutdown(); + _executor = null; + } + + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + + if (hostLBTimer != null) { + hostLBTimer.cancel(); + hostLBTimer = null; + } + + if (certTimer != null) { + certTimer.cancel(); + certTimer = null; + } } public Long getId() { @@ -318,6 +371,15 @@ public class Agent implements HandlerFactory, IAgentControl { _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id)); } + private synchronized void scheduleServicesRestartTask() { + if (certTimer != null) { + certTimer.cancel(); + certTimer.purge(); + } + certTimer = new Timer("Certificate Renewal Timer"); + certTimer.schedule(new PostCertificateRenewalTask(this), 5000L); + } + private synchronized void scheduleHostLBCheckerTask(final long checkInterval) { if (hostLBTimer != null) { hostLBTimer.cancel(); @@ -578,6 +640,9 @@ public class Agent implements HandlerFactory, IAgentControl { answer = setupAgentKeystore((SetupKeyStoreCommand) cmd); } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) { answer = setupAgentCertificate((SetupCertificateCommand) cmd); + if (Host.Type.Routing.equals(_resource.getType())) { + scheduleServicesRestartTask(); + } } else if (cmd instanceof SetupDirectDownloadCertificate) { answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd); } else if (cmd instanceof SetupMSListCommand) { @@ -641,7 +706,7 @@ public class Agent implements HandlerFactory, IAgentControl { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; String cerFile = agentFile.getParent() + "/" + certificateName + ".cer"; Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile)); @@ -666,13 +731,13 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCsrFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.CSR_FILENAME; - String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName); + String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (Strings.isNullOrEmpty(storedPassword)) { storedPassword = keyStorePassword; - _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword); + _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword); } Script script = new Script(true, _keystoreSetupPath, 60000, s_logger); @@ -706,10 +771,10 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCertFile; - final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultPrivateKeyFile; - final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCaCertFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME; + final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.PKEY_FILENAME; + final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.CACERT_FILENAME; try { FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset()); @@ -722,7 +787,7 @@ public class Agent implements HandlerFactory, IAgentControl { Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger); script.add(agentFile.getAbsolutePath()); script.add(keyStoreFile); - script.add(KeyStoreUtils.agentMode); + script.add(KeyStoreUtils.AGENT_MODE); script.add(certFile); script.add(""); script.add(caCertFile); @@ -1072,6 +1137,60 @@ public class Agent implements HandlerFactory, IAgentControl { } } + /** + * Task stops the current agent and launches a new agent + * when there are no outstanding jobs in the agent's task queue + */ + public class PostCertificateRenewalTask extends ManagedContextTimerTask { + + private Agent agent; + + public PostCertificateRenewalTask(final Agent agent) { + this.agent = agent; + } + + @Override + protected void runInContext() { + while (true) { + try { + if (_inProgress.get() == 0) { + s_logger.debug("Running post certificate renewal task to restart services."); + + // Let the resource perform any post certificate renewal cleanups + _resource.executeRequest(new PostCertificateRenewalCommand()); + + IAgentShell shell = agent._shell; + ServerResource resource = agent._resource.getClass().newInstance(); + + // Stop current agent + agent.cancelTasks(); + agent._reconnectAllowed = false; + Runtime.getRuntime().removeShutdownHook(agent._shutdownThread); + agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates"); + + // Nullify references for GC + agent._shell = null; + agent._watchList = null; + agent._shutdownThread = null; + agent._controlListeners = null; + agent = null; + + // Start a new agent instance + shell.launchNewAgent(resource); + return; + } + if (s_logger.isTraceEnabled()) { + s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds"); + } + Thread.sleep(5000); + } catch (final Exception e) { + s_logger.warn("Failed to execute post certificate renewal command:", e); + break; + } + } + } + } + public class PreferredHostCheckerTask extends ManagedContextTimerTask { @Override diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index 13b6c65a351..01654ac9caa 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -419,7 +419,7 @@ public class AgentShell implements IAgentShell, Daemon { final Constructor constructor = impl.getDeclaredConstructor(); constructor.setAccessible(true); ServerResource resource = (ServerResource)constructor.newInstance(); - launchAgent(getNextAgentId(), resource); + launchNewAgent(resource); } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { @@ -447,9 +447,10 @@ public class AgentShell implements IAgentShell, Daemon { s_logger.trace("Launching agent based on type=" + typeInfo); } - private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException { + public void launchNewAgent(ServerResource resource) throws ConfigurationException { // we don't track agent after it is launched for now - Agent agent = new Agent(this, localAgentId, resource); + _agents.clear(); + Agent agent = new Agent(this, getNextAgentId(), resource); _agents.add(agent); agent.start(); } diff --git a/agent/src/main/java/com/cloud/agent/IAgentShell.java b/agent/src/main/java/com/cloud/agent/IAgentShell.java index 5b52cee6361..5d389a07041 100644 --- a/agent/src/main/java/com/cloud/agent/IAgentShell.java +++ b/agent/src/main/java/com/cloud/agent/IAgentShell.java @@ -19,6 +19,9 @@ package com.cloud.agent; import java.util.Map; import java.util.Properties; +import javax.naming.ConfigurationException; + +import com.cloud.resource.ServerResource; import com.cloud.utils.backoff.BackoffAlgorithm; public interface IAgentShell { @@ -66,4 +69,6 @@ public interface IAgentShell { void updateConnectedHost(); String getConnectedHost(); + + void launchNewAgent(ServerResource resource) throws ConfigurationException; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index d7c3d56f9b9..0ffe8cc0ea2 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -157,11 +157,11 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s " + "%s %d " + "/usr/local/cloud/systemvm/conf/%s", - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_FILENAME, cmd.getKeystorePassword(), cmd.getValidityDays(), - KeyStoreUtils.defaultCsrFile); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args); + KeyStoreUtils.CSR_FILENAME); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args); return new SetupKeystoreAnswer(result.getDetails()); } @@ -171,15 +171,15 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\"", - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, cmd.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, cmd.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, cmd.getEncodedPrivateKey()); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args); return new SetupCertificateAnswer(result.isSuccess()); } diff --git a/core/src/main/java/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java b/core/src/main/java/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java new file mode 100644 index 00000000000..12df6196128 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.ca; + +import com.cloud.agent.api.Command; + +public class PostCertificateRenewalCommand extends Command { + + public PostCertificateRenewalCommand() { + super(); + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/ca/SetupCertificateCommand.java b/core/src/main/java/org/apache/cloudstack/ca/SetupCertificateCommand.java index 1cd31509d39..7727282bcee 100644 --- a/core/src/main/java/org/apache/cloudstack/ca/SetupCertificateCommand.java +++ b/core/src/main/java/org/apache/cloudstack/ca/SetupCertificateCommand.java @@ -82,15 +82,15 @@ public class SetupCertificateCommand extends NetworkElementCommand { } public String getEncodedPrivateKey() { - return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCertificate() { - return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCaCertificates() { - return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public boolean isHandleByAgent() { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index a9b8b687fab..c358c3ca680 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -50,6 +50,7 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu + ;; esac diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index 4a3585ac4e2..6584b35861a 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -241,7 +241,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con @Override public char[] getKeyStorePassphrase() { - return KeyStoreUtils.defaultKeystorePassphrase; + return KeyStoreUtils.DEFAULT_KS_PASSPHRASE; } ///////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java new file mode 100644 index 00000000000..df89d2470dd --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = PostCertificateRenewalCommand.class) +public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtPostCertificateRenewalCommandWrapper.class); + + @Override + public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) { + s_logger.info("Restarting libvirt after certificate provisioning/renewal"); + if (command != null) { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, s_logger); + if ("Ubuntu".equals(serverResource.getHostDistro()) || "Debian".equals(serverResource.getHostDistro())) { + script.add("libvirt-bin"); + } else { + script.add("libvirtd"); + } + script.add("restart"); + script.execute(); + return new SetupCertificateAnswer(true); + } + return new SetupCertificateAnswer(false); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dd039e54263..fc5e5395b87 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -41,11 +41,19 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; +import org.apache.cloudstack.utils.linux.CPUStat; +import org.apache.cloudstack.utils.linux.MemStat; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; @@ -68,14 +76,6 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import com.google.common.base.Strings; - -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; -import org.apache.cloudstack.utils.linux.CPUStat; -import org.apache.cloudstack.utils.linux.MemStat; -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -168,6 +168,7 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; /** * LibvirtComputingResource execute requests on the computing/routing host using @@ -239,6 +240,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected long _hypervisorLibvirtVersion; protected long _hypervisorQemuVersion; protected String _hypervisorPath; + protected String _hostDistro; protected String _networkDirectSourceMode; protected String _networkDirectDevice; protected String _sysvmISOPath; @@ -2599,11 +2601,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); cmd.setIqn(getIqn()); + if (cmd.getHostDetails().containsKey("Host.OS")) { + _hostDistro = cmd.getHostDetails().get("Host.OS"); + } + StartupStorageCommand sscmd = null; try { @@ -3777,4 +3784,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public long getTotalMemory() { return _totalMemory; } + + public String getHostDistro() { + return _hostDistro; + } + + public boolean isHostSecured() { + // Test for host certificates + final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); + if (confFile == null || !confFile.exists() || !new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists()) { + return false; + } + + // Test for libvirt TLS configuration + try { + new Connect(String.format("qemu+tls://%s/system", _privateIp)); + } catch (final LibvirtException ignored) { + return false; + } + return true; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index ad32759a517..67ec1b731af 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -20,8 +20,8 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,12 +46,10 @@ import javax.xml.transform.stream.StreamResult; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; - import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo.DomainState; import org.libvirt.LibvirtException; - import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; @@ -71,6 +69,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Strings; @ResourceWrapper(handles = MigrateCommand.class) public final class LibvirtMigrateCommandWrapper extends CommandWrapper { @@ -80,9 +79,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, command.isAutoConvergence(), vmName, command.getDestinationIp()); @@ -203,6 +210,9 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper { - private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class); + private static final Logger s_logger = Logger.getLogger(LibvirtModifyTargetsCommandWrapper.class); @Override public Answer execute(final ModifyTargetsCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index ed13cb25f0c..da71e40c30f 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -18,10 +18,14 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.junit.Test; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.exception.CloudRuntimeException; + public class LibvirtMigrateCommandWrapperTest { String fullfile = "\n" + @@ -303,4 +307,22 @@ public class LibvirtMigrateCommandWrapperTest { final String result = lw.replaceIpForVNCInDescFile(xmlDesc, targetIp); assertTrue("transformation does not live up to expectation:\n" + result, expectedXmlDesc.equals(result)); } + + @Test + public void testMigrationUri() { + final String ip = "10.1.1.1"; + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + LibvirtComputingResource lcr = new LibvirtComputingResource(); + if (lcr.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tls://%s/system", ip)); + } else { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tcp://%s/system", ip)); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testMigrationUriException() { + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + lw.createMigrationURI(null, new LibvirtComputingResource()); + } } diff --git a/python/lib/cloud_utils.py b/python/lib/cloud_utils.py index a9afb04dd8e..36ce617b3f1 100644 --- a/python/lib/cloud_utils.py +++ b/python/lib/cloud_utils.py @@ -802,9 +802,9 @@ class SetupFirewall(ConfigTask): rule = "-p tcp -m tcp --dport 16509 -j ACCEPT" if rule in iptablessave().stdout: return True return False - + def execute(self): - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() if distro in (Fedora , CentOS, RHEL6): for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT') o = service.iptables.save() ; print o.stdout + o.stderr diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index 68d1b9cef5b..2b27868db05 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -471,6 +471,23 @@ class securityPolicyConfigRedhat(serviceCfgBase): logging.debug(formatExceptionInfo()) return False +def configureLibvirtConfig(tls_enabled = True, cfg = None): + cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg) + if tls_enabled: + cfo.addEntry("listen_tcp", "0") + cfo.addEntry("listen_tls", "1") + cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"") + cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"") + cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"") + else: + cfo.addEntry("listen_tcp", "1") + cfo.addEntry("listen_tls", "0") + cfo.addEntry("tcp_port", "\"16509\"") + cfo.addEntry("tls_port", "\"16514\"") + cfo.addEntry("auth_tcp", "\"none\"") + cfo.addEntry("auth_tls", "\"none\"") + cfo.save() + class libvirtConfigRedhat(serviceCfgBase): def __init__(self, syscfg): super(libvirtConfigRedhat, self).__init__(syscfg) @@ -478,12 +495,7 @@ class libvirtConfigRedhat(serviceCfgBase): def config(self): try: - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\"") - cfo.addEntry("auth_tcp", "\"none\"") - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) cfo = configFileOps("/etc/sysconfig/libvirtd", self) cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'") @@ -516,12 +528,7 @@ class libvirtConfigUbuntu(serviceCfgBase): self.serviceName = "Libvirt" def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) if os.path.exists("/etc/init/libvirt-bin.conf"): cfo = configFileOps("/etc/init/libvirt-bin.conf", self) @@ -567,7 +574,7 @@ class firewallConfigUbuntu(serviceCfgBase): def config(self): try: - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() for p in ports: bash("ufw allow %s"%p) bash("ufw allow proto tcp from any to any port 5900:6100") @@ -627,7 +634,7 @@ class firewallConfigBase(serviceCfgBase): class firewallConfigAgent(firewallConfigBase): def __init__(self, syscfg): super(firewallConfigAgent, self).__init__(syscfg) - self.ports = "22 16509 5900:6100 49152:49216".split() + self.ports = "22 16509 16514 5900:6100 49152:49216".split() if syscfg.env.distribution.getVersion() == "CentOS": self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"] else: diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index 67ce3400345..96196d93902 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" +LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -78,6 +79,18 @@ fi rm -f "$NEW_KS_FILE.p12" mv -f "$NEW_KS_FILE" "$KS_FILE" +# Secure libvirtd on cert import +if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/CA + mkdir -p /etc/pki/libvirt/private + ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem + cloudstack-setup-agent -s > /dev/null +fi + # Update ca-certs if we're in systemvm if [ -f "$SYSTEM_FILE" ]; then mkdir -p /usr/local/share/ca-certificates/cloudstack diff --git a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index ef69fdcb436..0b8b40b1b9b 100644 --- a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.discoverer; import java.net.InetAddress; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -141,11 +142,6 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements } private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) { - if (!caManager.canProvisionCertificates()) { - s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate"); - return; - } - if (sshConnection == null) { throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp); } @@ -161,17 +157,17 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s " + "%s %d " + "/etc/cloudstack/agent/%s", - KeyStoreUtils.keyStoreSetupScript, - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_SETUP_SCRIPT, + KeyStoreUtils.KS_FILENAME, PasswordGenerator.generateRandomPassword(16), validityPeriod, - KeyStoreUtils.defaultCsrFile)); + KeyStoreUtils.CSR_FILENAME)); if (!keystoreSetupResult.isSuccess()) { throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp); } - final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null); + final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null); if (certificate == null || certificate.getClientCertificate() == null) { throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp); } @@ -184,14 +180,14 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\"", - KeyStoreUtils.keyStoreImportScript, - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_IMPORT_SCRIPT, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, certificateCommand.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, certificateCommand.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, certificateCommand.getEncodedPrivateKey())); if (setupCertResult != null && !setupCertResult.isSuccess()) { @@ -288,9 +284,13 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic; } + if (!caManager.canProvisionCertificates()) { + throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency."); + } + setupAgentSecurity(sshConnection, agentIp, hostname); - String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a"; + String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a -s "; parameters += " --pubNic=" + kvmPublicNic; parameters += " --prvNic=" + kvmPrivateNic; diff --git a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java index 02e7a4231a4..49484f668a1 100644 --- a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java @@ -27,7 +27,6 @@ import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -191,8 +190,7 @@ public class CAManagerImpl extends ManagerBase implements CAManager { if (Strings.isNullOrEmpty(csr)) { return false; } - final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()), - Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); + final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); } catch (final AgentUnavailableException | OperationTimedoutException e) { LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e); diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index e24a27790b7..d2a26c988b0 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -585,7 +585,7 @@ routing_svcs() { systemctl enable haproxy echo "haproxy apache2" > /var/cache/cloud/enabled_svcs echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs - if [ $RROUTER -eq 1 ] + if [ "$RROUTER" -eq "1" ] then systemctl disable --now dnsmasq systemctl enable conntrackd diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3504849786b..d882c1fc9ae 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -21,9 +21,11 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.cloudstackAPI import (recoverVirtualMachine, destroyVirtualMachine, attachIso, - detachIso) -from marvin.lib.utils import (cleanup_resources, - validateList) + detachIso, + provisionCertificate, + updateConfiguration) +from marvin.lib.utils import * + from marvin.lib.base import (Account, ServiceOffering, VirtualMachine, @@ -33,11 +35,13 @@ from marvin.lib.base import (Account, Configurations) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_hosts) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr #Import System modules import time +import re _multiprocess_shared_ = True class TestDeployVM(cloudstackTestCase): @@ -781,3 +785,301 @@ class TestVMLifeCycle(cloudstackTestCase): "Check if ISO is detached from virtual machine" ) return + +class TestSecuredVmMigration(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSecuredVmMigration, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"] + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls._cleanup = [ + cls.small_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + + cls.apiclient = super(TestSecuredVmMigration, cls).getClsTestClient().getApiClient() + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + self.updateConfiguration("ca.plugin.root.auth.strictness", "false") + self.make_all_hosts_secure() + + if self.hypervisor.lower() not in ["kvm"]: + self.skipTest("Secured migration is not supported on other than KVM") + + def tearDown(self): + self.make_all_hosts_secure() + + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_secured_vm_migration(self): + """Test secured VM migration""" + + # Validate the following + # 1. Environment has enough hosts for migration + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + + hosts = self.get_hosts() + + secured_hosts = [] + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + if len(secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + + origin_host = secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tls') + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_not_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + #self.skipTest() + # Validate the following + # 1. Prepare 2 hosts to run in non-secured more + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + hosts = self.get_hosts() + for host in hosts: + self.make_unsecure_connection(host) + + non_secured_hosts = [] + + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'false': + non_secured_hosts.append(host) + + if len(non_secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + origin_host = non_secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='false', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tcp') + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_secured_to_nonsecured_vm_migration(self): + """Test destroy Virtual Machine + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to a Secured host + # 3. Migrates the VM to the non-secured host and assers the migration is via TCP. + + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(secured_hosts[0]) + try: + self.migrate_and_check(origin_host=secured_hosts[0], destination_host=non_secured_host, proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_nonsecured_to_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to the non-secured host + # 3. Migrates the VM to the secured host and assers the migration is via TCP. + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + + hosts = self.get_hosts() + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(non_secured_host) + + try: + self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp') + except Exception: + pass + else: + self.fail("Migration succeed, instead it should fail") + + def get_target_host(self, secured, virtualmachineid): + target_hosts = Host.listForMigration(self.apiclient, + virtualmachineid=virtualmachineid) + for host in target_hosts: + h = list_hosts(self.apiclient,type='Routing', id=host.id)[0] + if h.details.secured == secured: + return h + + cloudstackTestCase.skipTest(self, "No target hosts available, skipping test.") + + def check_migration_protocol(self, protocol, host): + resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("grep -a Live /var/log/cloudstack/agent/agent.log | tail -1") + + if protocol not in resp[0]: + cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n" + "Instead we got: " + resp[0]) + + def make_unsecure_connection(self, host): + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("rm -f /etc/cloudstack/agent/cloud*") + + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i 's/listen_tls.*/listen_tls=0/g' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf ") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i '/.*_file.*/d' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service libvirtd restart") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service cloudstack-agent restart") + + self.check_connection(host=host, secured='false') + time.sleep(10) + return host + + def make_all_hosts_secure(self): + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + for host in hosts: + cmd = provisionCertificate.provisionCertificateCmd() + cmd.hostid = host.id + self.apiclient.updateConfiguration(cmd) + + for host in hosts: + self.check_connection(secured='true', host=host) + + def get_hosts(self): + + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed") + return hosts + + def deploy_vm(self, origin_host): + return VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id, + mode=self.services["mode"], + hostid=origin_host.id + ) + + def check_connection(self, secured, host, retries=5, interval=5): + + while retries > -1: + time.sleep(interval) + host = Host.list( + self.apiclient, + zoneid=self.zone.id, + hostid=host.id, + type='Routing' + )[0] + if host.details.secured != secured: + if retries >= 0: + retries = retries - 1 + continue + else: + return + + raise Exception("Host communication is not as expected: " + secured + + ". Instead it's: " + host.details.secured) + + def migrate_and_check(self, origin_host, destination_host, proto): + + self.vm_to_migrate.migrate(self.apiclient, hostid=destination_host.id) + + self.check_migration_protocol(protocol=proto, host=origin_host) + + vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0] + + self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM") + + def updateConfiguration(self, name, value): + cmd = updateConfiguration.updateConfigurationCmd() + cmd.name = name + cmd.value = value + self.apiclient.updateConfiguration(cmd) \ No newline at end of file diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index aaeba9a3b32..ca85bb0bee3 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -12670,11 +12670,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it background-position: -101px -647px; } +.secureKVMHost .icon, .resetPassword .icon, .changePassword .icon { background-position: -68px -30px; } +.secureKVMHost:hover .icon, .resetPassword:hover .icon, .changePassword:hover .icon { background-position: -68px -612px; diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 342e8838284..2bb127db922 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -276,6 +276,7 @@ var dictionary = { "label.action.restore.instance.processing":"Restoring Instance....", "label.action.revert.snapshot":"Revert to Snapshot", "label.action.revert.snapshot.processing":"Reverting to Snapshot...", +"label.action.secure.host":"Provision Host Security Keys", "label.action.start.instance":"Start Instance", "label.action.start.instance.processing":"Starting Instance....", "label.action.start.router":"Start Router", @@ -1920,6 +1921,7 @@ var dictionary = { "message.action.reset.password.warning":"Your instance must be stopped before attempting to change its current password.", "message.action.restore.instance":"Please confirm that you want to restore this instance.", "message.action.revert.snapshot":"Please confirm that you want to revert the owning volume to this snapshot.", +"message.action.secure.host":"This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm?", "message.action.start.instance":"Please confirm that you want to start this instance.", "message.action.start.router":"Please confirm that you want to start this router.", "message.action.start.systemvm":"Please confirm that you want to start this system VM.", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 582e394baca..950f447cacf 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -9198,6 +9198,11 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } + }); } @@ -15710,7 +15715,8 @@ 'Down': 'off', 'Disconnected': 'off', 'Alert': 'off', - 'Error': 'off' + 'Error': 'off', + 'Unsecure': 'warning' } }, powerstate: { @@ -15758,6 +15764,10 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } }); } @@ -16527,6 +16537,40 @@ } }, + secureKVMHost: { + label: 'label.action.secure.host', + action: function(args) { + var data = { + hostid: args.context.hosts[0].id + }; + $.ajax({ + url: createURL('provisionCertificate'), + data: data, + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.provisioncertificateresponse.jobid, + getActionFilter: function () { + return hostActionfilter; + } + } + }); + } + }); + }, + messages: { + confirm: function (args) { + return 'message.action.secure.host'; + }, + notification: function (args) { + return 'label.action.secure.host'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, enableMaintenanceMode: { label: 'label.action.enable.maintenance.mode', @@ -21924,6 +21968,11 @@ if (jsonObj.state != "Disconnected") allowedActions.push("forceReconnect"); + + if (jsonObj.hypervisor == "KVM") { + allowedActions.push("secureKVMHost"); + } + } else if (jsonObj.resourcestate == "ErrorInMaintenance") { allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 25f6662c522..65824408770 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -379,7 +379,7 @@ public class Link { return caService.createSSLEngine(sslContext, clientAddress); } s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine"); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase); final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); @@ -409,11 +409,11 @@ public class Link { } public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException { - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; File confFile = PropertiesUtil.findConfigFile("agent.properties"); if (confFile != null) { s_logger.info("Conf file found: " + confFile.getAbsolutePath()); - final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.passphrasePropertyName); + final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (pass != null) { passphrase = pass.toCharArray(); } @@ -421,7 +421,7 @@ public class Link { InputStream stream = null; if (confFile != null) { - final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; if (new File(keystorePath).exists()) { stream = new FileInputStream(keystorePath); } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 7087395b050..35aa24b1a84 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -37,6 +37,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import org.joda.time.Duration; @@ -202,7 +203,7 @@ public class Script implements Callable { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command)); + _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]); } try { diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 10407b65642..5324cdcbc18 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -139,7 +139,7 @@ public class SSHCmdHelper { } public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException { - s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]); + s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]); Session sshSession = null; try { sshSession = sshConnection.openSession(); @@ -202,7 +202,7 @@ public class SSHCmdHelper { final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString()); if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) { - s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); + s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); } // exit status delivery might get delayed diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java index 8259d77d4ff..c6f8d21918c 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java @@ -19,23 +19,34 @@ package org.apache.cloudstack.utils.security; +import java.io.File; + +import com.cloud.utils.PropertiesUtil; + public class KeyStoreUtils { + public static final String KS_SETUP_SCRIPT = "keystore-setup"; + public static final String KS_IMPORT_SCRIPT = "keystore-cert-import"; - public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks"; - public static String defaultKeystoreFile = "cloud.jks"; - public static String defaultPrivateKeyFile = "cloud.key"; - public static String defaultCsrFile = "cloud.csr"; - public static String defaultCertFile = "cloud.crt"; - public static String defaultCaCertFile = "cloud.ca.crt"; - public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray(); + public static final String AGENT_PROPSFILE = "agent.properties"; + public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase"; - public static String certNewlineEncoder = "^"; - public static String certSpaceEncoder = "~"; + public static final String KS_FILENAME = "cloud.jks"; + public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray(); - public static String keyStoreSetupScript = "keystore-setup"; - public static String keyStoreImportScript = "keystore-cert-import"; - public static String passphrasePropertyName = "keystore.passphrase"; + public static final String CACERT_FILENAME = "cloud.ca.crt"; + public static final String CERT_FILENAME = "cloud.crt"; + public static final String CSR_FILENAME = "cloud.csr"; + public static final String PKEY_FILENAME = "cloud.key"; - public static String sshMode = "ssh"; - public static String agentMode = "agent"; + public static final String CERT_NEWLINE_ENCODER = "^"; + public static final String CERT_SPACE_ENCODER = "~"; + + public static final String SSH_MODE = "ssh"; + public static final String AGENT_MODE = "agent"; + public static final String SECURED = "secured"; + + public static boolean isHostSecured() { + final File confFile = PropertiesUtil.findConfigFile("agent.properties"); + return confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + CERT_FILENAME).exists(); + } }