Merge branch '4.11'

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

agent/bindir/cloud-setup-agent.in

@@ -26,6 +26,7 @@ from cloudutils.configFileOps import  configFileOps
 from cloudutils.globalEnv import globalEnv
 from cloudutils.networkConfig import networkConfig
 from cloudutils.syscfg import sysConfigFactory
+from cloudutils.serviceConfig import configureLibvirtConfig
 
 from optparse import OptionParser
 
@@ -100,6 +101,7 @@ if __name__ == '__main__':
     parser.add_option("-c", "--cluster", dest="cluster", help="cluster id")
     parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type")
     parser.add_option("-g", "--guid", dest="guid", help="guid")
+    parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd")
     parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface")
     parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface")
     parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface")
@@ -110,6 +112,12 @@ if __name__ == '__main__':
         glbEnv.bridgeType = bridgeType
 
     (options, args) = parser.parse_args()
+
+    if not options.auto and options.secure:
+        configureLibvirtConfig(True)
+        print "Libvirtd with TLS configured"
+        sys.exit(0)
+
     if options.auto is None:
         userInputs = getUserInputs()
         glbEnv.mgtSvr = userInputs[0]
@@ -138,7 +146,9 @@ if __name__ == '__main__':
         glbEnv.nics.append(options.prvNic)
         glbEnv.nics.append(options.pubNic)
         glbEnv.nics.append(options.guestNic)
-        
+
+    glbEnv.secure = options.secure
+
     print "Starting to configure your system:"
     syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     try:

agent/src/main/java/com/cloud/agent/Agent.java

@@ -42,6 +42,7 @@ import javax.naming.ConfigurationException;
 import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
 import org.apache.cloudstack.agent.lb.SetupMSListAnswer;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
+import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
 import org.apache.cloudstack.ca.SetupCertificateAnswer;
 import org.apache.cloudstack.ca.SetupCertificateCommand;
 import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@@ -68,6 +69,7 @@ import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.transport.Request;
 import com.cloud.agent.transport.Response;
 import com.cloud.exception.AgentControlChannelException;
+import com.cloud.host.Host;
 import com.cloud.resource.ServerResource;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.StringUtils;
@@ -127,6 +129,7 @@ public class Agent implements HandlerFactory, IAgentControl {
     Long _id;
 
     Timer _timer = new Timer("Agent Timer");
+    Timer certTimer;
     Timer hostLBTimer;
 
     List<WatchTask> _watchList = new ArrayList<WatchTask>();
@@ -140,9 +143,11 @@ public class Agent implements HandlerFactory, IAgentControl {
     long _startupWait = _startupWaitDefault;
     boolean _reconnectAllowed = true;
     //For time sentitive task, e.g. PingTask
-    private final ThreadPoolExecutor _ugentTaskPool;
+    ThreadPoolExecutor _ugentTaskPool;
     ExecutorService _executor;
 
+    Thread _shutdownThread = new ShutdownThread(this);
+
     private String _keystoreSetupPath;
     private String _keystoreCertImportPath;
 
@@ -153,7 +158,7 @@ public class Agent implements HandlerFactory, IAgentControl {
 
         _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this);
 
-        Runtime.getRuntime().addShutdownHook(new ShutdownThread(this));
+        Runtime.getRuntime().addShutdownHook(_shutdownThread);
 
         _ugentTaskPool =
                 new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue<Runnable>(), new NamedThreadFactory(
@@ -192,7 +197,7 @@ public class Agent implements HandlerFactory, IAgentControl {
         // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp());
 
         s_logger.debug("Adding shutdown hook");
-        Runtime.getRuntime().addShutdownHook(new ShutdownThread(this));
+        Runtime.getRuntime().addShutdownHook(_shutdownThread);
 
         _ugentTaskPool =
                 new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue<Runnable>(), new NamedThreadFactory(
@@ -239,20 +244,39 @@ public class Agent implements HandlerFactory, IAgentControl {
         return _resource.getClass().getSimpleName();
     }
 
+    /**
+     * In case of a software based agent restart, this method
+     * can help to perform explicit garbage collection of any old
+     * agent instances and its inner objects.
+     */
+    private void scavengeOldAgentObjects() {
+        _executor.submit(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    Thread.sleep(2000L);
+                } catch (final InterruptedException ignored) {
+                } finally {
+                    System.gc();
+                }
+            }
+        });
+    }
+
     public void start() {
         if (!_resource.start()) {
             s_logger.error("Unable to start the resource: " + _resource.getName());
             throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName());
         }
 
-        _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript);
+        _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT);
         if (_keystoreSetupPath == null) {
-            throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript));
+            throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT));
         }
 
-        _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript);
+        _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT);
         if (_keystoreCertImportPath == null) {
-            throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript));
+            throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT));
         }
 
         try {
@@ -274,6 +298,7 @@ public class Agent implements HandlerFactory, IAgentControl {
             }
         }
         _shell.updateConnectedHost();
+        scavengeOldAgentObjects();
     }
 
     public void stop(final String reason, final String detail) {
@@ -298,6 +323,7 @@ public class Agent implements HandlerFactory, IAgentControl {
             }
             _connection.stop();
             _connection = null;
+            _link = null;
         }
 
         if (_resource != null) {
@@ -305,7 +331,34 @@ public class Agent implements HandlerFactory, IAgentControl {
             _resource = null;
         }
 
-        _ugentTaskPool.shutdownNow();
+        if (_startup != null) {
+            _startup = null;
+        }
+
+        if (_ugentTaskPool != null) {
+            _ugentTaskPool.shutdownNow();
+            _ugentTaskPool = null;
+        }
+
+        if (_executor != null) {
+            _executor.shutdown();
+            _executor = null;
+        }
+
+        if (_timer != null) {
+            _timer.cancel();
+            _timer = null;
+        }
+
+        if (hostLBTimer != null) {
+            hostLBTimer.cancel();
+            hostLBTimer = null;
+        }
+
+        if (certTimer != null) {
+            certTimer.cancel();
+            certTimer = null;
+        }
     }
 
     public Long getId() {
@@ -318,6 +371,15 @@ public class Agent implements HandlerFactory, IAgentControl {
         _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id));
     }
 
+    private synchronized void scheduleServicesRestartTask() {
+        if (certTimer != null) {
+            certTimer.cancel();
+            certTimer.purge();
+        }
+        certTimer = new Timer("Certificate Renewal Timer");
+        certTimer.schedule(new PostCertificateRenewalTask(this), 5000L);
+    }
+
     private synchronized void scheduleHostLBCheckerTask(final long checkInterval) {
         if (hostLBTimer != null) {
             hostLBTimer.cancel();
@@ -578,6 +640,9 @@ public class Agent implements HandlerFactory, IAgentControl {
                         answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
                     } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
                         answer = setupAgentCertificate((SetupCertificateCommand) cmd);
+                        if (Host.Type.Routing.equals(_resource.getType())) {
+                            scheduleServicesRestartTask();
+                        }
                     } else if (cmd instanceof SetupDirectDownloadCertificate) {
                         answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
                     } else if (cmd instanceof SetupMSListCommand) {
@@ -641,7 +706,7 @@ public class Agent implements HandlerFactory, IAgentControl {
             return new Answer(cmd, false, "Failed to find agent.properties file");
         }
 
-        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME;
 
         String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
         Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile));
@@ -666,13 +731,13 @@ public class Agent implements HandlerFactory, IAgentControl {
         if (agentFile == null) {
             return new Answer(cmd, false, "Failed to find agent.properties file");
         }
-        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME;
-        final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCsrFile;
+        final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.CSR_FILENAME;
 
-        String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName);
+        String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY);
         if (Strings.isNullOrEmpty(storedPassword)) {
             storedPassword = keyStorePassword;
-            _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword);
+            _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword);
         }
 
         Script script = new Script(true, _keystoreSetupPath, 60000, s_logger);
@@ -706,10 +771,10 @@ public class Agent implements HandlerFactory, IAgentControl {
         if (agentFile == null) {
             return new Answer(cmd, false, "Failed to find agent.properties file");
         }
-        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME;
-        final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCertFile;
+        final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME;
-        final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultPrivateKeyFile;
+        final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.PKEY_FILENAME;
-        final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCaCertFile;
+        final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.CACERT_FILENAME;
 
         try {
             FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset());
@@ -722,7 +787,7 @@ public class Agent implements HandlerFactory, IAgentControl {
         Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger);
         script.add(agentFile.getAbsolutePath());
         script.add(keyStoreFile);
-        script.add(KeyStoreUtils.agentMode);
+        script.add(KeyStoreUtils.AGENT_MODE);
         script.add(certFile);
         script.add("");
         script.add(caCertFile);
@@ -1072,6 +1137,60 @@ public class Agent implements HandlerFactory, IAgentControl {
         }
     }
 
+    /**
+     * Task stops the current agent and launches a new agent
+     * when there are no outstanding jobs in the agent's task queue
+     */
+    public class PostCertificateRenewalTask extends ManagedContextTimerTask {
+
+        private Agent agent;
+
+        public PostCertificateRenewalTask(final Agent agent) {
+            this.agent = agent;
+        }
+
+        @Override
+        protected void runInContext() {
+            while (true) {
+                try {
+                    if (_inProgress.get() == 0) {
+                        s_logger.debug("Running post certificate renewal task to restart services.");
+
+                        // Let the resource perform any post certificate renewal cleanups
+                        _resource.executeRequest(new PostCertificateRenewalCommand());
+
+                        IAgentShell shell = agent._shell;
+                        ServerResource resource = agent._resource.getClass().newInstance();
+
+                        // Stop current agent
+                        agent.cancelTasks();
+                        agent._reconnectAllowed = false;
+                        Runtime.getRuntime().removeShutdownHook(agent._shutdownThread);
+                        agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates");
+
+                        // Nullify references for GC
+                        agent._shell = null;
+                        agent._watchList = null;
+                        agent._shutdownThread = null;
+                        agent._controlListeners = null;
+                        agent = null;
+
+                        // Start a new agent instance
+                        shell.launchNewAgent(resource);
+                        return;
+                    }
+                    if (s_logger.isTraceEnabled()) {
+                        s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds");
+                    }
+                    Thread.sleep(5000);
+                } catch (final Exception e) {
+                    s_logger.warn("Failed to execute post certificate renewal command:", e);
+                    break;
+                }
+            }
+        }
+    }
+
     public class PreferredHostCheckerTask extends ManagedContextTimerTask {
 
         @Override

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -419,7 +419,7 @@ public class AgentShell implements IAgentShell, Daemon {
                 final Constructor<?> constructor = impl.getDeclaredConstructor();
                 constructor.setAccessible(true);
                 ServerResource resource = (ServerResource)constructor.newInstance();
-                launchAgent(getNextAgentId(), resource);
+                launchNewAgent(resource);
             } catch (final ClassNotFoundException e) {
                 throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString());
             } catch (final SecurityException e) {
@@ -447,9 +447,10 @@ public class AgentShell implements IAgentShell, Daemon {
         s_logger.trace("Launching agent based on type=" + typeInfo);
     }
 
-    private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException {
+    public void launchNewAgent(ServerResource resource) throws ConfigurationException {
         // we don't track agent after it is launched for now
-        Agent agent = new Agent(this, localAgentId, resource);
+        _agents.clear();
+        Agent agent = new Agent(this, getNextAgentId(), resource);
         _agents.add(agent);
         agent.start();
     }

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -19,6 +19,9 @@ package com.cloud.agent;
 import java.util.Map;
 import java.util.Properties;
 
+import javax.naming.ConfigurationException;
+
+import com.cloud.resource.ServerResource;
 import com.cloud.utils.backoff.BackoffAlgorithm;
 
 public interface IAgentShell {
@@ -66,4 +69,6 @@ public interface IAgentShell {
     void updateConnectedHost();
 
     String getConnectedHost();
+
+    void launchNewAgent(ServerResource resource) throws ConfigurationException;
 }

core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java

@@ -157,11 +157,11 @@ public class VirtualRoutingResource {
                         "/usr/local/cloud/systemvm/conf/%s " +
                         "%s %d " +
                         "/usr/local/cloud/systemvm/conf/%s",
-                KeyStoreUtils.defaultKeystoreFile,
+                KeyStoreUtils.KS_FILENAME,
                 cmd.getKeystorePassword(),
                 cmd.getValidityDays(),
-                KeyStoreUtils.defaultCsrFile);
+                KeyStoreUtils.CSR_FILENAME);
-        ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args);
+        ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args);
         return new SetupKeystoreAnswer(result.getDetails());
     }
 
@@ -171,15 +171,15 @@ public class VirtualRoutingResource {
                         "/usr/local/cloud/systemvm/conf/%s \"%s\" " +
                         "/usr/local/cloud/systemvm/conf/%s \"%s\" " +
                         "/usr/local/cloud/systemvm/conf/%s \"%s\"",
-                KeyStoreUtils.defaultKeystoreFile,
+                KeyStoreUtils.KS_FILENAME,
-                KeyStoreUtils.sshMode,
+                KeyStoreUtils.SSH_MODE,
-                KeyStoreUtils.defaultCertFile,
+                KeyStoreUtils.CERT_FILENAME,
                 cmd.getEncodedCertificate(),
-                KeyStoreUtils.defaultCaCertFile,
+                KeyStoreUtils.CACERT_FILENAME,
                 cmd.getEncodedCaCertificates(),
-                KeyStoreUtils.defaultPrivateKeyFile,
+                KeyStoreUtils.PKEY_FILENAME,
                 cmd.getEncodedPrivateKey());
-        ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args);
+        ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args);
         return new SetupCertificateAnswer(result.isSuccess());
     }
 

core/src/main/java/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java

@@ -0,0 +1,34 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.ca;
+
+import com.cloud.agent.api.Command;
+
+public class PostCertificateRenewalCommand extends Command {
+
+    public PostCertificateRenewalCommand() {
+        super();
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/org/apache/cloudstack/ca/SetupCertificateCommand.java

@@ -82,15 +82,15 @@ public class SetupCertificateCommand extends NetworkElementCommand {
     }
 
     public String getEncodedPrivateKey() {
-        return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder);
+        return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER);
     }
 
     public String getEncodedCertificate() {
-        return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder);
+        return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER);
     }
 
     public String getEncodedCaCertificates() {
-        return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder);
+        return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER);
     }
 
     public boolean isHandleByAgent() {

debian/cloudstack-agent.postinst

@@ -50,6 +50,7 @@ case "$1" in
             mkdir /etc/libvirt/hooks
         fi
         cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu
+
         ;;
 esac
 

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java

@@ -241,7 +241,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
 
     @Override
     public char[] getKeyStorePassphrase() {
-        return KeyStoreUtils.defaultKeystorePassphrase;
+        return KeyStoreUtils.DEFAULT_KS_PASSPHRASE;
     }
 
     /////////////////////////////////////////////////

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
+import org.apache.cloudstack.ca.SetupCertificateAnswer;
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.utils.script.Script;
+
+@ResourceWrapper(handles =  PostCertificateRenewalCommand.class)
+public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper<PostCertificateRenewalCommand, Answer, LibvirtComputingResource> {
+
+    private static final Logger s_logger = Logger.getLogger(LibvirtPostCertificateRenewalCommandWrapper.class);
+
+    @Override
+    public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) {
+        s_logger.info("Restarting libvirt after certificate provisioning/renewal");
+        if (command != null) {
+            final int timeout = 30000;
+            Script script = new Script(true, "service", timeout, s_logger);
+            if ("Ubuntu".equals(serverResource.getHostDistro()) || "Debian".equals(serverResource.getHostDistro())) {
+                script.add("libvirt-bin");
+            } else {
+               script.add("libvirtd");
+            }
+            script.add("restart");
+            script.execute();
+            return new SetupCertificateAnswer(true);
+       }
+        return new SetupCertificateAnswer(false);
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -41,11 +41,19 @@ import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+
 import javax.naming.ConfigurationException;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
+import org.apache.cloudstack.utils.linux.CPUStat;
+import org.apache.cloudstack.utils.linux.MemStat;
+import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
+import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.ArrayUtils;
@@ -68,14 +76,6 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
-import com.google.common.base.Strings;
-
-import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
-import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
-import org.apache.cloudstack.utils.linux.CPUStat;
-import org.apache.cloudstack.utils.linux.MemStat;
-import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
@@ -168,6 +168,7 @@ import com.cloud.utils.ssh.SshHelper;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.PowerState;
 import com.cloud.vm.VmDetailConstants;
+import com.google.common.base.Strings;
 
 /**
  * LibvirtComputingResource execute requests on the computing/routing host using
@@ -239,6 +240,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     protected long _hypervisorLibvirtVersion;
     protected long _hypervisorQemuVersion;
     protected String _hypervisorPath;
+    protected String _hostDistro;
     protected String _networkDirectSourceMode;
     protected String _networkDirectDevice;
     protected String _sysvmISOPath;
@@ -2599,11 +2601,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         fillNetworkInformation(cmd);
         _privateIp = cmd.getPrivateIpAddress();
         cmd.getHostDetails().putAll(getVersionStrings());
+        cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase());
         cmd.setPool(_pool);
         cmd.setCluster(_clusterId);
         cmd.setGatewayIpAddress(_localGateway);
         cmd.setIqn(getIqn());
 
+        if (cmd.getHostDetails().containsKey("Host.OS")) {
+            _hostDistro = cmd.getHostDetails().get("Host.OS");
+        }
+
         StartupStorageCommand sscmd = null;
         try {
 
@@ -3777,4 +3784,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     public long getTotalMemory() {
         return _totalMemory;
     }
+
+    public String getHostDistro() {
+        return _hostDistro;
+    }
+
+    public boolean isHostSecured() {
+        // Test for host certificates
+        final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE);
+        if (confFile == null || !confFile.exists() || !new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists()) {
+            return false;
+        }
+
+        // Test for libvirt TLS configuration
+        try {
+            new Connect(String.format("qemu+tls://%s/system", _privateIp));
+        } catch (final LibvirtException ignored) {
+            return false;
+        }
+        return true;
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java

@@ -20,8 +20,8 @@
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -46,12 +46,10 @@ import javax.xml.transform.stream.StreamResult;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Logger;
-
 import org.libvirt.Connect;
 import org.libvirt.Domain;
 import org.libvirt.DomainInfo.DomainState;
 import org.libvirt.LibvirtException;
-
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NamedNodeMap;
@@ -71,6 +69,7 @@ import com.cloud.resource.CommandWrapper;
 import com.cloud.resource.ResourceWrapper;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Strings;
 
 @ResourceWrapper(handles =  MigrateCommand.class)
 public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCommand, Answer, LibvirtComputingResource> {
@@ -80,9 +79,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
     private static final String CONTENTS_WILDCARD = "(?s).*";
     private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class);
 
+    protected String createMigrationURI(final String destinationIp, final LibvirtComputingResource libvirtComputingResource) {
+        if (Strings.isNullOrEmpty(destinationIp)) {
+            throw new CloudRuntimeException("Provided libvirt destination ip is invalid");
+        }
+        return String.format("%s://%s/system", libvirtComputingResource.isHostSecured() ? "qemu+tls" : "qemu+tcp", destinationIp);
+    }
+
     @Override
     public Answer execute(final MigrateCommand command, final LibvirtComputingResource libvirtComputingResource) {
         final String vmName = command.getVmName();
+        final String destinationUri = createMigrationURI(command.getDestinationIp(), libvirtComputingResource);
 
         String result = null;
 
@@ -140,10 +147,10 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
                 xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage);
             }
 
-            dconn = libvirtUtilitiesHelper.retrieveQemuConnection("qemu+tcp://" + command.getDestinationIp() + "/system");
+            dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri);
 
             //run migration in thread so we can monitor it
-            s_logger.info("Live migration of instance " + vmName + " initiated");
+            s_logger.info("Live migration of instance " + vmName + " initiated to destination host: " + dconn.getURI());
             final ExecutorService executor = Executors.newFixedThreadPool(1);
             final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage,
                     command.isAutoConvergence(), vmName, command.getDestinationIp());
@@ -203,6 +210,9 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
         } catch (final LibvirtException e) {
             s_logger.debug("Can't migrate domain: " + e.getMessage());
             result = e.getMessage();
+            if (result.startsWith("unable to connect to server") && result.endsWith("refused")) {
+                result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility and firewall rules on the source and destination hosts.", destinationUri);
+            }
         } catch (final InterruptedException e) {
             s_logger.debug("Interrupted while migrating domain: " + e.getMessage());
             result = e.getMessage();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java

@@ -38,7 +38,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
 
 @ResourceWrapper(handles =  ModifyTargetsCommand.class)
 public final class LibvirtModifyTargetsCommandWrapper extends CommandWrapper<ModifyTargetsCommand, Answer, LibvirtComputingResource> {
-    private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class);
+    private static final Logger s_logger = Logger.getLogger(LibvirtModifyTargetsCommandWrapper.class);
 
     @Override
     public Answer execute(final ModifyTargetsCommand command, final LibvirtComputingResource libvirtComputingResource) {

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java

@@ -18,10 +18,14 @@
 //
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
 
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 public class LibvirtMigrateCommandWrapperTest {
     String fullfile =
 "<domain type='kvm' id='4'>\n" +
@@ -303,4 +307,22 @@ public class LibvirtMigrateCommandWrapperTest {
         final String result = lw.replaceIpForVNCInDescFile(xmlDesc, targetIp);
         assertTrue("transformation does not live up to expectation:\n" + result, expectedXmlDesc.equals(result));
     }
+
+    @Test
+    public void testMigrationUri() {
+        final String ip = "10.1.1.1";
+        LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
+        LibvirtComputingResource lcr = new LibvirtComputingResource();
+        if (lcr.isHostSecured()) {
+            assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tls://%s/system", ip));
+        } else {
+            assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tcp://%s/system", ip));
+        }
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testMigrationUriException() {
+        LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
+        lw.createMigrationURI(null, new LibvirtComputingResource());
+    }
 }

python/lib/cloud_utils.py

@@ -802,9 +802,9 @@ class SetupFirewall(ConfigTask):
 		rule = "-p tcp -m tcp --dport 16509 -j ACCEPT"
 		if rule in iptablessave().stdout: return True
 		return False
-	
+
 	def execute(self):
-		ports = "22 1798 16509".split()
+		ports = "22 1798 16509 16514".split()
 		if distro in (Fedora , CentOS, RHEL6):
 			for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT')
 			o = service.iptables.save() ; print o.stdout + o.stderr

python/lib/cloudutils/serviceConfig.py

@@ -471,6 +471,23 @@ class securityPolicyConfigRedhat(serviceCfgBase):
             logging.debug(formatExceptionInfo())
             return False
 
+def configureLibvirtConfig(tls_enabled = True, cfg = None):
+    cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg)
+    if tls_enabled:
+        cfo.addEntry("listen_tcp", "0")
+        cfo.addEntry("listen_tls", "1")
+        cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"")
+        cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"")
+        cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"")
+    else:
+        cfo.addEntry("listen_tcp", "1")
+        cfo.addEntry("listen_tls", "0")
+    cfo.addEntry("tcp_port", "\"16509\"")
+    cfo.addEntry("tls_port", "\"16514\"")
+    cfo.addEntry("auth_tcp", "\"none\"")
+    cfo.addEntry("auth_tls", "\"none\"")
+    cfo.save()
+
 class libvirtConfigRedhat(serviceCfgBase):
     def __init__(self, syscfg):
         super(libvirtConfigRedhat, self).__init__(syscfg)
@@ -478,12 +495,7 @@ class libvirtConfigRedhat(serviceCfgBase):
 
     def config(self):
         try:
-            cfo = configFileOps("/etc/libvirt/libvirtd.conf", self)
+            configureLibvirtConfig(self.syscfg.env.secure, self)
-            cfo.addEntry("listen_tcp", "1")
-            cfo.addEntry("tcp_port", "\"16509\"")
-            cfo.addEntry("auth_tcp", "\"none\"")
-            cfo.addEntry("listen_tls", "0")
-            cfo.save()
 
             cfo = configFileOps("/etc/sysconfig/libvirtd", self)
             cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'")
@@ -516,12 +528,7 @@ class libvirtConfigUbuntu(serviceCfgBase):
         self.serviceName = "Libvirt"
 
     def setupLiveMigration(self):
-        cfo = configFileOps("/etc/libvirt/libvirtd.conf", self)
+        configureLibvirtConfig(self.syscfg.env.secure, self)
-        cfo.addEntry("listen_tcp", "1")
-        cfo.addEntry("tcp_port", "\"16509\"");
-        cfo.addEntry("auth_tcp", "\"none\"");
-        cfo.addEntry("listen_tls", "0")
-        cfo.save()
 
         if os.path.exists("/etc/init/libvirt-bin.conf"):
             cfo = configFileOps("/etc/init/libvirt-bin.conf", self)
@@ -567,7 +574,7 @@ class firewallConfigUbuntu(serviceCfgBase):
 
     def config(self):
         try:
-            ports = "22 1798 16509".split()
+            ports = "22 1798 16509 16514".split()
             for p in ports:
                 bash("ufw allow %s"%p)
             bash("ufw allow proto tcp from any to any port 5900:6100")
@@ -627,7 +634,7 @@ class firewallConfigBase(serviceCfgBase):
 class firewallConfigAgent(firewallConfigBase):
     def __init__(self, syscfg):
         super(firewallConfigAgent, self).__init__(syscfg)
-        self.ports = "22 16509 5900:6100 49152:49216".split()
+        self.ports = "22 16509 16514 5900:6100 49152:49216".split()
         if syscfg.env.distribution.getVersion() == "CentOS":
             self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"]
         else:

scripts/util/keystore-cert-import

@@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ')
 
 ALIAS="cloud"
 SYSTEM_FILE="/var/cache/cloud/cmdline"
+LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf"
 
 # Find keystore password
 KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null  | sed 's/keystore.passphrase=//g' 2>/dev/null)
@@ -78,6 +79,18 @@ fi
 rm -f "$NEW_KS_FILE.p12"
 mv -f "$NEW_KS_FILE" "$KS_FILE"
 
+# Secure libvirtd on cert import
+if [ -f "$LIBVIRTD_FILE" ]; then
+    mkdir -p /etc/pki/CA
+    mkdir -p /etc/pki/libvirt/private
+    ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem
+    ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem
+    ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem
+    ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem
+    ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem
+    cloudstack-setup-agent -s > /dev/null
+fi
+
 # Update ca-certs if we're in systemvm
 if [ -f "$SYSTEM_FILE" ]; then
     mkdir -p /usr/local/share/ca-certificates/cloudstack

server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java

@@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.discoverer;
 
 import java.net.InetAddress;
 import java.net.URI;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -141,11 +142,6 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
     }
 
     private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) {
-        if (!caManager.canProvisionCertificates()) {
-            s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate");
-            return;
-        }
-
         if (sshConnection == null) {
             throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp);
         }
@@ -161,17 +157,17 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
                                 "/etc/cloudstack/agent/%s " +
                                 "%s %d " +
                                 "/etc/cloudstack/agent/%s",
-                        KeyStoreUtils.keyStoreSetupScript,
+                        KeyStoreUtils.KS_SETUP_SCRIPT,
-                        KeyStoreUtils.defaultKeystoreFile,
+                        KeyStoreUtils.KS_FILENAME,
                         PasswordGenerator.generateRandomPassword(16),
                         validityPeriod,
-                        KeyStoreUtils.defaultCsrFile));
+                        KeyStoreUtils.CSR_FILENAME));
 
         if (!keystoreSetupResult.isSuccess()) {
             throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp);
         }
 
-        final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null);
+        final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null);
         if (certificate == null || certificate.getClientCertificate() == null) {
             throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp);
         }
@@ -184,14 +180,14 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
                                     "/etc/cloudstack/agent/%s \"%s\" " +
                                     "/etc/cloudstack/agent/%s \"%s\" " +
                                     "/etc/cloudstack/agent/%s \"%s\"",
-                            KeyStoreUtils.keyStoreImportScript,
+                            KeyStoreUtils.KS_IMPORT_SCRIPT,
-                            KeyStoreUtils.defaultKeystoreFile,
+                            KeyStoreUtils.KS_FILENAME,
-                            KeyStoreUtils.sshMode,
+                            KeyStoreUtils.SSH_MODE,
-                            KeyStoreUtils.defaultCertFile,
+                            KeyStoreUtils.CERT_FILENAME,
                             certificateCommand.getEncodedCertificate(),
-                            KeyStoreUtils.defaultCaCertFile,
+                            KeyStoreUtils.CACERT_FILENAME,
                             certificateCommand.getEncodedCaCertificates(),
-                            KeyStoreUtils.defaultPrivateKeyFile,
+                            KeyStoreUtils.PKEY_FILENAME,
                             certificateCommand.getEncodedPrivateKey()));
 
         if (setupCertResult != null && !setupCertResult.isSuccess()) {
@@ -288,9 +284,13 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
                 kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic;
             }
 
+            if (!caManager.canProvisionCertificates()) {
+                throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency.");
+            }
+
             setupAgentSecurity(sshConnection, agentIp, hostname);
 
-            String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId     + " -c " + clusterId + " -g " + guid + " -a";
+            String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId     + " -c " + clusterId + " -g " + guid + " -a -s ";
 
             parameters += " --pubNic=" + kvmPublicNic;
             parameters += " --prvNic=" + kvmPrivateNic;

server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java

@@ -27,7 +27,6 @@ import java.security.cert.CertificateNotYetValidException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -191,8 +190,7 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
             if (Strings.isNullOrEmpty(csr)) {
                 return false;
             }
-            final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()),
+            final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider);
-                    Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider);
             return deployCertificate(host, certificate, reconnect, null);
         } catch (final AgentUnavailableException | OperationTimedoutException e) {
             LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e);

systemvm/debian/opt/cloud/bin/setup/common.sh

@@ -585,7 +585,7 @@ routing_svcs() {
    systemctl enable haproxy
    echo "haproxy apache2" > /var/cache/cloud/enabled_svcs
    echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs
-   if [ $RROUTER -eq 1 ]
+   if [ "$RROUTER" -eq "1" ]
    then
        systemctl disable --now dnsmasq
        systemctl enable conntrackd

test/integration/smoke/test_vm_life_cycle.py

@@ -21,9 +21,11 @@ from marvin.cloudstackTestCase import cloudstackTestCase
 from marvin.cloudstackAPI import (recoverVirtualMachine,
                                   destroyVirtualMachine,
                                   attachIso,
-                                  detachIso)
+                                  detachIso,
-from marvin.lib.utils import (cleanup_resources,
+                                  provisionCertificate,
-                              validateList)
+                                  updateConfiguration)
+from marvin.lib.utils import *
+
 from marvin.lib.base import (Account,
                              ServiceOffering,
                              VirtualMachine,
@@ -33,11 +35,13 @@ from marvin.lib.base import (Account,
                              Configurations)
 from marvin.lib.common import (get_domain,
                                 get_zone,
-                                get_template)
+                                get_template,
+                               list_hosts)
 from marvin.codes import FAILED, PASS
 from nose.plugins.attrib import attr
 #Import System modules
 import time
+import re
 
 _multiprocess_shared_ = True
 class TestDeployVM(cloudstackTestCase):
@@ -781,3 +785,301 @@ class TestVMLifeCycle(cloudstackTestCase):
                          "Check if ISO is detached from virtual machine"
                          )
         return
+
+class TestSecuredVmMigration(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        testClient = super(TestSecuredVmMigration, cls).getClsTestClient()
+        cls.apiclient = testClient.getApiClient()
+        cls.services = testClient.getParsedTestDataConfig()
+        cls.hypervisor = testClient.getHypervisorInfo()
+
+        # Get Zone, Domain and templates
+        domain = get_domain(cls.apiclient)
+        cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.services['mode'] = cls.zone.networktype
+        cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__
+        cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"]
+
+        template = get_template(
+                            cls.apiclient,
+                            cls.zone.id,
+                            cls.services["ostype"]
+                            )
+        if template == FAILED:
+            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+
+        # Set Zones and disk offerings
+        cls.services["small"]["zoneid"] = cls.zone.id
+        cls.services["small"]["template"] = template.id
+
+        cls.services["iso1"]["zoneid"] = cls.zone.id
+
+        # Create VMs, NAT Rules etc
+        cls.account = Account.create(
+                            cls.apiclient,
+                            cls.services["account"],
+                            domainid=domain.id
+                            )
+
+        cls.small_offering = ServiceOffering.create(
+                                    cls.apiclient,
+                                    cls.services["service_offerings"]["small"]
+                                    )
+
+        cls._cleanup = [
+                        cls.small_offering,
+                        cls.account
+                        ]
+
+    @classmethod
+    def tearDownClass(cls):
+
+        cls.apiclient = super(TestSecuredVmMigration, cls).getClsTestClient().getApiClient()
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.cleanup = []
+        self.updateConfiguration("ca.plugin.root.auth.strictness", "false")
+        self.make_all_hosts_secure()
+
+        if self.hypervisor.lower() not in ["kvm"]:
+            self.skipTest("Secured migration is not supported on other than KVM")
+
+    def tearDown(self):
+        self.make_all_hosts_secure()
+
+        try:
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_01_secured_vm_migration(self):
+        """Test secured VM migration"""
+
+        # Validate the following
+        # 1. Environment has enough hosts for migration
+        # 2. DeployVM on suitable host (with another host in the cluster)
+        # 3. Migrate the VM and assert migration successful
+
+        hosts = self.get_hosts()
+
+        secured_hosts = []
+
+        for host in hosts:
+            if host.details.secured == 'true':
+                secured_hosts.append(host)
+
+        if len(secured_hosts) < 2:
+            self.skipTest("At least two hosts should be present in the zone for migration")
+
+        origin_host = secured_hosts[0]
+
+        self.vm_to_migrate = self.deploy_vm(origin_host)
+
+        target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id)
+
+        self.migrate_and_check(origin_host, target_host, proto='tls')
+
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_02_not_secured_vm_migration(self):
+        """Test Non-secured VM Migration
+        """
+        #self.skipTest()
+        # Validate the following
+        # 1. Prepare 2 hosts to run in non-secured more
+        # 2. DeployVM on suitable host (with another host in the cluster)
+        # 3. Migrate the VM and assert migration successful
+        hosts = self.get_hosts()
+        for host in hosts:
+            self.make_unsecure_connection(host)
+
+        non_secured_hosts = []
+
+        hosts = self.get_hosts()
+
+        for host in hosts:
+            if host.details.secured == 'false':
+                non_secured_hosts.append(host)
+
+        if len(non_secured_hosts) < 2:
+            self.skipTest("At least two hosts should be present in the zone for migration")
+        origin_host = non_secured_hosts[0]
+
+        self.vm_to_migrate = self.deploy_vm(origin_host)
+
+        target_host = self.get_target_host(secured='false', virtualmachineid=self.vm_to_migrate.id)
+
+        self.migrate_and_check(origin_host, target_host, proto='tcp')
+
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_03_secured_to_nonsecured_vm_migration(self):
+        """Test destroy Virtual Machine
+        """
+
+        # Validate the following
+        # 1. Makes one of the hosts non-secured
+        # 2. Deploys a VM to a Secured host
+        # 3. Migrates the VM to the non-secured host and assers the migration is via TCP.
+
+        hosts = self.get_hosts()
+
+        non_secured_host = self.make_unsecure_connection(hosts[0])
+
+        secured_hosts = []
+        hosts = self.get_hosts()
+
+        for host in hosts:
+            if host.details.secured == 'true':
+                secured_hosts.append(host)
+
+        self.vm_to_migrate = self.deploy_vm(secured_hosts[0])
+        try:
+            self.migrate_and_check(origin_host=secured_hosts[0], destination_host=non_secured_host, proto='tcp')
+        except Exception:
+            pass
+        else: self.fail("Migration succeed, instead it should fail")
+
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_04_nonsecured_to_secured_vm_migration(self):
+        """Test Non-secured VM Migration
+        """
+
+        # Validate the following
+        # 1. Makes one of the hosts non-secured
+        # 2. Deploys a VM to the non-secured host
+        # 3. Migrates the VM to the secured host and assers the migration is via TCP.
+        hosts = self.get_hosts()
+
+        non_secured_host = self.make_unsecure_connection(hosts[0])
+
+        secured_hosts = []
+
+        hosts = self.get_hosts()
+        for host in hosts:
+            if host.details.secured == 'true':
+                secured_hosts.append(host)
+
+        self.vm_to_migrate = self.deploy_vm(non_secured_host)
+
+        try:
+            self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp')
+        except Exception:
+            pass
+        else:
+            self.fail("Migration succeed, instead it should fail")
+
+    def get_target_host(self, secured, virtualmachineid):
+        target_hosts = Host.listForMigration(self.apiclient,
+                                             virtualmachineid=virtualmachineid)
+        for host in target_hosts:
+            h = list_hosts(self.apiclient,type='Routing', id=host.id)[0]
+            if h.details.secured == secured:
+                return h
+
+        cloudstackTestCase.skipTest(self, "No target hosts available, skipping test.")
+
+    def check_migration_protocol(self, protocol, host):
+        resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("grep -a Live /var/log/cloudstack/agent/agent.log | tail -1")
+
+        if protocol not in resp[0]:
+            cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n"
+                                    "Instead we got: " + resp[0])
+
+    def make_unsecure_connection(self, host):
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("rm -f /etc/cloudstack/agent/cloud*")
+
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("sed -i 's/listen_tls.*/listen_tls=0/g' /etc/libvirt/libvirtd.conf")
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf ")
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("sed -i '/.*_file.*/d' /etc/libvirt/libvirtd.conf")
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("service libvirtd restart")
+        SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\
+            .execute("service cloudstack-agent restart")
+
+        self.check_connection(host=host, secured='false')
+        time.sleep(10)
+        return host
+
+    def make_all_hosts_secure(self):
+        hosts = Host.list(
+            self.apiclient,
+            zoneid=self.zone.id,
+            type='Routing'
+        )
+        for host in hosts:
+            cmd = provisionCertificate.provisionCertificateCmd()
+            cmd.hostid = host.id
+            self.apiclient.updateConfiguration(cmd)
+
+        for host in hosts:
+            self.check_connection(secured='true', host=host)
+
+    def get_hosts(self):
+
+        hosts = Host.list(
+            self.apiclient,
+            zoneid=self.zone.id,
+            type='Routing'
+        )
+        self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed")
+        return hosts
+
+    def deploy_vm(self, origin_host):
+        return VirtualMachine.create(
+            self.apiclient,
+            self.services["small"],
+            accountid=self.account.name,
+            domainid=self.account.domainid,
+            serviceofferingid=self.small_offering.id,
+            mode=self.services["mode"],
+            hostid=origin_host.id
+        )
+
+    def check_connection(self, secured, host, retries=5, interval=5):
+
+        while retries > -1:
+            time.sleep(interval)
+            host = Host.list(
+                self.apiclient,
+                zoneid=self.zone.id,
+                hostid=host.id,
+                type='Routing'
+            )[0]
+            if host.details.secured != secured:
+                if retries >= 0:
+                    retries = retries - 1
+                    continue
+            else:
+                return
+
+        raise Exception("Host communication is not as expected: " + secured +
+                        ". Instead it's: " + host.details.secured)
+
+    def migrate_and_check(self, origin_host, destination_host, proto):
+
+        self.vm_to_migrate.migrate(self.apiclient, hostid=destination_host.id)
+
+        self.check_migration_protocol(protocol=proto, host=origin_host)
+
+        vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0]
+
+        self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM")
+
+    def updateConfiguration(self, name, value):
+        cmd = updateConfiguration.updateConfigurationCmd()
+        cmd.name = name
+        cmd.value = value
+        self.apiclient.updateConfiguration(cmd)

ui/css/cloudstack3.css

@@ -12670,11 +12670,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it
   background-position: -101px -647px;
 }
 
+.secureKVMHost .icon,
 .resetPassword .icon,
 .changePassword .icon {
   background-position: -68px -30px;
 }
 
+.secureKVMHost:hover .icon,
 .resetPassword:hover .icon,
 .changePassword:hover .icon {
   background-position: -68px -612px;

ui/l10n/en.js

@@ -276,6 +276,7 @@ var dictionary = {
 "label.action.restore.instance.processing":"Restoring Instance....",
 "label.action.revert.snapshot":"Revert to Snapshot",
 "label.action.revert.snapshot.processing":"Reverting to Snapshot...",
+"label.action.secure.host":"Provision Host Security Keys",
 "label.action.start.instance":"Start Instance",
 "label.action.start.instance.processing":"Starting Instance....",
 "label.action.start.router":"Start Router",
@@ -1920,6 +1921,7 @@ var dictionary = {
 "message.action.reset.password.warning":"Your instance must be stopped before attempting to change its current password.",
 "message.action.restore.instance":"Please confirm that you want to restore this instance.",
 "message.action.revert.snapshot":"Please confirm that you want to revert the owning volume to this snapshot.",
+"message.action.secure.host":"This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm?",
 "message.action.start.instance":"Please confirm that you want to start this instance.",
 "message.action.start.router":"Please confirm that you want to start this router.",
 "message.action.start.systemvm":"Please confirm that you want to start this system VM.",

ui/scripts/system.js

@@ -9198,6 +9198,11 @@
                                                     if (host && host.outofbandmanagement) {
                                                         items[idx].powerstate = host.outofbandmanagement.powerstate;
                                                     }
+
+                                                    if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') {
+                                                        items[idx].state = 'Unsecure';
+                                                    }
+
                                                 });
                                             }
 
@@ -15710,7 +15715,8 @@
                                 'Down': 'off',
                                 'Disconnected': 'off',
                                 'Alert': 'off',
-                                'Error': 'off'
+                                'Error': 'off',
+                                'Unsecure': 'warning'
                             }
                         },
                         powerstate: {
@@ -15758,6 +15764,10 @@
                                         if (host && host.outofbandmanagement) {
                                             items[idx].powerstate = host.outofbandmanagement.powerstate;
                                         }
+
+                                        if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') {
+                                            items[idx].state = 'Unsecure';
+                                        }
                                     });
                                 }
 
@@ -16527,6 +16537,40 @@
                                 }
                             },
 
+                            secureKVMHost: {
+                                label: 'label.action.secure.host',
+                                action: function(args) {
+                                    var data = {
+                                        hostid: args.context.hosts[0].id
+                                    };
+                                    $.ajax({
+                                        url: createURL('provisionCertificate'),
+                                        data: data,
+                                        async: true,
+                                        success: function(json) {
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: json.provisioncertificateresponse.jobid,
+                                                    getActionFilter: function () {
+                                                        return hostActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        }
+                                    });
+                                },
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.action.secure.host';
+                                    },
+                                    notification: function (args) {
+                                        return 'label.action.secure.host';
+                                    }
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
+                            },
 
                             enableMaintenanceMode: {
                                 label: 'label.action.enable.maintenance.mode',
@@ -21924,6 +21968,11 @@
 
             if (jsonObj.state != "Disconnected")
             allowedActions.push("forceReconnect");
+
+            if (jsonObj.hypervisor == "KVM") {
+                allowedActions.push("secureKVMHost");
+            }
+
         } else if (jsonObj.resourcestate == "ErrorInMaintenance") {
             allowedActions.push("edit");
             allowedActions.push("enableMaintenanceMode");

utils/src/main/java/com/cloud/utils/nio/Link.java

@@ -379,7 +379,7 @@ public class Link {
             return caService.createSSLEngine(sslContext, clientAddress);
         }
         s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine");
-        char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase;
+        char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE;
         final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase);
         final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
         final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
@@ -409,11 +409,11 @@ public class Link {
     }
 
     public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException {
-        char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase;
+        char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE;
         File confFile = PropertiesUtil.findConfigFile("agent.properties");
         if (confFile != null) {
             s_logger.info("Conf file found: " + confFile.getAbsolutePath());
-            final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.passphrasePropertyName);
+            final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY);
             if (pass != null) {
                 passphrase = pass.toCharArray();
             }
@@ -421,7 +421,7 @@ public class Link {
 
         InputStream stream = null;
         if (confFile != null) {
-            final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+            final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME;
             if (new File(keystorePath).exists()) {
                 stream = new FileInputStream(keystorePath);
             }

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -37,6 +37,7 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Logger;
 import org.joda.time.Duration;
@@ -202,7 +203,7 @@ public class Script implements Callable<String> {
         String[] command = _command.toArray(new String[_command.size()]);
 
         if (_logger.isDebugEnabled()) {
-            _logger.debug("Executing: " + buildCommandLine(command));
+            _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]);
         }
 
         try {

utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java

@@ -139,7 +139,7 @@ public class SSHCmdHelper {
     }
 
     public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException {
-        s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]);
+        s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]);
         Session sshSession = null;
         try {
             sshSession = sshConnection.openSession();
@@ -202,7 +202,7 @@ public class SSHCmdHelper {
 
             final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString());
             if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) {
-                s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr());
+                s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr());
             }
 
             // exit status delivery might get delayed

utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java

@@ -19,23 +19,34 @@
 
 package org.apache.cloudstack.utils.security;
 
+import java.io.File;
+
+import com.cloud.utils.PropertiesUtil;
+
 public class KeyStoreUtils {
+    public static final String KS_SETUP_SCRIPT = "keystore-setup";
+    public static final String KS_IMPORT_SCRIPT = "keystore-cert-import";
 
-    public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks";
+    public static final String AGENT_PROPSFILE = "agent.properties";
-    public static String defaultKeystoreFile = "cloud.jks";
+    public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase";
-    public static String defaultPrivateKeyFile = "cloud.key";
-    public static String defaultCsrFile = "cloud.csr";
-    public static String defaultCertFile = "cloud.crt";
-    public static String defaultCaCertFile = "cloud.ca.crt";
-    public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray();
 
-    public static String certNewlineEncoder = "^";
+    public static final String KS_FILENAME = "cloud.jks";
-    public static String certSpaceEncoder = "~";
+    public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray();
 
-    public static String keyStoreSetupScript = "keystore-setup";
+    public static final String CACERT_FILENAME = "cloud.ca.crt";
-    public static String keyStoreImportScript = "keystore-cert-import";
+    public static final String CERT_FILENAME = "cloud.crt";
-    public static String passphrasePropertyName = "keystore.passphrase";
+    public static final String CSR_FILENAME = "cloud.csr";
+    public static final String PKEY_FILENAME = "cloud.key";
 
-    public static String sshMode = "ssh";
+    public static final String CERT_NEWLINE_ENCODER = "^";
-    public static String agentMode = "agent";
+    public static final String CERT_SPACE_ENCODER = "~";
+
+    public static final String SSH_MODE = "ssh";
+    public static final String AGENT_MODE = "agent";
+    public static final String SECURED = "secured";
+
+    public static boolean isHostSecured() {
+        final File confFile = PropertiesUtil.findConfigFile("agent.properties");
+        return confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + CERT_FILENAME).exists();
+    }
 }