Dont add host back after agent service restart (#4228)

agent/src/main/java/com/cloud/agent/Agent.java

@@ -420,6 +420,19 @@ public class Agent implements HandlerFactory, IAgentControl {
         }
     }
 
+    /**
+     * Cleanup agent zone properties.
+     *
+     * Unset zone, cluster and pod values so that host is not added back
+     * when service is restarted. This will be set to proper values
+     * when host is added back
+     */
+    protected void cleanupAgentZoneProperties() {
+        _shell.setPersistentProperty(null, "zone", "");
+        _shell.setPersistentProperty(null, "cluster", "");
+        _shell.setPersistentProperty(null, "pod", "");
+    }
+
     public synchronized void lockStartupTask(final Link link) {
         _startup = new StartupTask(link);
         _timer.schedule(_startup, _startupWait);
@@ -603,6 +616,9 @@ public class Agent implements HandlerFactory, IAgentControl {
                         final ShutdownCommand shutdown = (ShutdownCommand)cmd;
                         s_logger.debug("Received shutdownCommand, due to: " + shutdown.getReason());
                         cancelTasks();
+                        if (shutdown.isRemoveHost()) {
+                            cleanupAgentZoneProperties();
+                        }
                         _reconnectAllowed = false;
                         answer = new Answer(cmd, true, null);
                     } else if (cmd instanceof ReadyCommand && ((ReadyCommand)cmd).getDetails() != null) {

core/src/main/java/com/cloud/agent/api/ShutdownCommand.java

@@ -30,6 +30,7 @@ public class ShutdownCommand extends Command {
 
     private String reason;
     private String detail;
+    private boolean removeHost;
 
     protected ShutdownCommand() {
         super();
@@ -41,6 +42,13 @@ public class ShutdownCommand extends Command {
         this.detail = detail;
     }
 
+    public ShutdownCommand(String reason, String detail, boolean removeHost) {
+        super();
+        this.reason = reason;
+        this.detail = detail;
+        this.removeHost = removeHost;
+    }
+
     /**
      * @return return the reason the agent shutdown.  If Unknown, call getDetail() for any details.
      */
@@ -52,6 +60,10 @@ public class ShutdownCommand extends Command {
         return detail;
     }
 
+    public boolean isRemoveHost() {
+        return removeHost;
+    }
+
     @Override
     public boolean executeInSequence() {
         return true;

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -411,6 +411,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public final static ConfigKey<Long> IOPS_MAX_WRITE_LENGTH = new ConfigKey<Long>(Long.class, "vm.disk.iops.maximum.write.length", "Advanced", "0",
             "Maximum IOPS write burst duration (seconds). If '0' (zero) then does not check for maximum burst length.", true, ConfigKey.Scope.Global, null);
 
+    public static final ConfigKey<Boolean> ADD_HOST_ON_SERVICE_RESTART_KVM = new ConfigKey<Boolean>(Boolean.class, "add.host.on.service.restart.kvm", "Advanced", "true",
+            "Indicates whether the host will be added back to cloudstack after restarting agent service on host. If false it wont be added back even after service restart",
+            true, ConfigKey.Scope.Global, null);
+
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";
     private static final String BYTES_READ_RATE = "Bytes Read";
@@ -6381,6 +6385,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {SystemVMUseLocalStorage, IOPS_MAX_READ_LENGTH, IOPS_MAX_WRITE_LENGTH, BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH};
+        return new ConfigKey<?>[] {SystemVMUseLocalStorage, IOPS_MAX_READ_LENGTH, IOPS_MAX_WRITE_LENGTH,
+                BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM};
     }
 }

server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java

@@ -67,6 +67,8 @@ import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.ssh.SSHCmdHelper;
 import com.trilead.ssh2.Connection;
 
+import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVICE_RESTART_KVM;
+
 public abstract class LibvirtServerDiscoverer extends DiscovererBase implements Discoverer, Listener, ResourceStateAdapter {
     private static final Logger s_logger = Logger.getLogger(LibvirtServerDiscoverer.class);
     private final int _waitTime = 5; /* wait for 5 minutes */
@@ -348,6 +350,7 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
             _hostDao.saveDetails(connectedHost);
             return resources;
         } catch (DiscoveredWithErrorException e) {
+            s_logger.error("DiscoveredWithErrorException caught and rethrowing, message: "+ e.getMessage());
             throw e;
         } catch (Exception e) {
             String msg = " can't setup agent, due to " + e.toString() + " - " + e.getMessage();
@@ -474,7 +477,7 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
 
         _resourceMgr.deleteRoutingHost(host, isForced, isForceDeleteStorage);
         try {
-            ShutdownCommand cmd = new ShutdownCommand(ShutdownCommand.DeleteHost, null);
+            ShutdownCommand cmd = new ShutdownCommand(ShutdownCommand.DeleteHost, null, !ADD_HOST_ON_SERVICE_RESTART_KVM.value());
             agentMgr.send(host.getId(), cmd);
         } catch (AgentUnavailableException e) {
             s_logger.warn("Sending ShutdownCommand failed: ", e);