server: Use ACPI event to reboot VM on KVM, and Use 'forced' reboot option to stop and start the VM(s) (#4681)

* Updated libvirt's native reboot operation for VM on KVM using ACPI event, and Added 'forced' reboot option to stop and start the VM (using rebootVirtualMachine API)

* Added 'forced' reboot option for System VM and Router

- New parameter 'forced' in rebootSystemVm API, to stop and then start System VM
- New parameter 'forced' in rebootRouter API, to force stop and then start Router

* Added force reboot tests for User VM, System VM and Router

api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java

@@ -44,7 +44,7 @@ public interface VirtualNetworkApplianceService {
      *            the command specifying router's id
      * @return router if successful
      */
-    VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
+    VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork, boolean forced) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
 
     VirtualRouter upgradeRouter(UpgradeRouterCmd cmd);
 

api/src/main/java/org/apache/cloudstack/api/command/admin/router/RebootRouterCmd.java

@@ -49,6 +49,9 @@ public class RebootRouterCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, required = true, description = "the ID of the router")
     private Long id;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the router (Router is force Stopped and then Started)", since = "4.16.0")
+    private Boolean forced;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -96,10 +99,14 @@ public class RebootRouterCmd extends BaseAsyncCmd {
         return getId();
     }
 
+    public boolean isForced() {
+        return (forced != null) ? forced : false;
+    }
+
     @Override
     public void execute() throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
         CallContext.current().setEventDetails("Router Id: " + this._uuidMgr.getUuid(VirtualMachine.class,getId()));
-        VirtualRouter result = _routerService.rebootRouter(getId(), true);
+        VirtualRouter result = _routerService.rebootRouter(getId(), true, isForced());
         if (result != null) {
             DomainRouterResponse response = _responseGenerator.createDomainRouterResponse(result);
             response.setResponseName("router");

api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/RebootSystemVmCmd.java

@@ -52,6 +52,9 @@ public class RebootSystemVmCmd extends BaseAsyncCmd {
                description = "The ID of the system virtual machine")
     private Long id;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the system VM (System VM is Stopped and then Started)", since = "4.16.0")
+    private Boolean forced;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -104,6 +107,10 @@ public class RebootSystemVmCmd extends BaseAsyncCmd {
         return getId();
     }
 
+    public boolean isForced() {
+        return (forced != null) ? forced : false;
+    }
+
     @Override
     public void execute() {
         CallContext.current().setEventDetails("Vm Id: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()));

api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java

@@ -53,6 +53,9 @@ public class RebootVMCmd extends BaseAsyncCmd implements UserCmd {
             required=true, description="The ID of the virtual machine")
     private Long id;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the VM (VM is Stopped and then Started)", since = "4.16.0")
+    private Boolean forced;
+
     @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup menu or not", since = "4.15.0.0")
     private Boolean bootIntoSetup;
 
@@ -64,6 +67,10 @@ public class RebootVMCmd extends BaseAsyncCmd implements UserCmd {
         return id;
     }
 
+    public boolean isForced() {
+        return (forced != null) ? forced : false;
+    }
+
     public Boolean getBootIntoSetup() {
         return bootIntoSetup;
     }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -3458,7 +3458,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         final DeployDestination dest = new DeployDestination(dc, pod, cluster, host);
 
         try {
-
             final Commands cmds = new Commands(Command.OnError.Stop);
             RebootCommand rebootCmd = new RebootCommand(vm.getInstanceName(), getExecuteInSequence(vm.getHypervisorType()));
             VirtualMachineTO vmTo = getVmTO(vm.getId());
@@ -3476,7 +3475,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 }
                 return;
             }
-            s_logger.info("Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? " no reboot answer" : rebootAnswer.getDetails()));
+
+            String errorMsg = "Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? "no reboot response" : rebootAnswer.getDetails());
+            s_logger.info(errorMsg);
+            throw new CloudRuntimeException(errorMsg);
         } catch (final OperationTimedoutException e) {
             s_logger.warn("Unable to send the reboot command to host " + dest.getHost() + " for the vm " + vm + " due to operation timeout", e);
             throw new CloudRuntimeException("Failed to reboot the vm on host " + dest.getHost());

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -134,7 +134,6 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.GuestDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.GuestResourceDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InputDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
-import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.GuestNetType;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef.RngBackendModel;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SCSIDef;
@@ -3228,35 +3227,15 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         String msg = null;
         try {
             dm = conn.domainLookupByName(vmName);
-            // Get XML Dump including the secure information such as VNC password
+            // Perform ACPI based reboot
-            // By passing 1, or VIR_DOMAIN_XML_SECURE flag
+            // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainReboot
-            // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainXMLFlags
+            // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainRebootFlagValues
-            String vmDef = dm.getXMLDesc(1);
+            // Send ACPI event to Reboot
-            final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
+            dm.reboot(0x1);
-            parser.parseDomainXML(vmDef);
-            for (final InterfaceDef nic : parser.getInterfaces()) {
-                if (nic.getNetType() == GuestNetType.BRIDGE && nic.getBrName().startsWith("cloudVirBr")) {
-                    try {
-                        final int vnetId = Integer.parseInt(nic.getBrName().replaceFirst("cloudVirBr", ""));
-                        final String pifName = getPif(_guestBridgeName);
-                        final String newBrName = "br" + pifName + "-" + vnetId;
-                        vmDef = vmDef.replace("'" + nic.getBrName() + "'", "'" + newBrName + "'");
-                        s_logger.debug("VM bridge name is changed from " + nic.getBrName() + " to " + newBrName);
-                    } catch (final NumberFormatException e) {
-                        continue;
-                    }
-                }
-            }
-            s_logger.debug(vmDef);
-            msg = stopVM(conn, vmName, false);
-            msg = startVM(conn, vmName, vmDef);
             return null;
         } catch (final LibvirtException e) {
             s_logger.warn("Failed to create vm", e);
             msg = e.getMessage();
-        } catch (final InternalErrorException e) {
-            s_logger.warn("Failed to create vm", e);
-            msg = e.getMessage();
         } finally {
             try {
                 if (dm != null) {

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -510,7 +510,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true)
-    public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException,
+    public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork, final boolean forced) throws ConcurrentOperationException, ResourceUnavailableException,
     InsufficientCapacityException {
         final Account caller = CallContext.current().getCallingAccount();
 
@@ -531,7 +531,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         final UserVO user = _userDao.findById(CallContext.current().getCallingUserId());
         s_logger.debug("Stopping and starting router " + router + " as a part of router reboot");
 
-        if (stop(router, false, user, caller) != null) {
+        if (stop(router, forced, user, caller) != null) {
             return startRouter(routerId, reprogramNetwork);
         } else {
             throw new CloudRuntimeException("Failed to reboot router " + router);
@@ -1048,7 +1048,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
                 }
                 _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, backupRouter.getDataCenterId(), backupRouter.getPodIdToDeployIn(), title, title);
                 try {
-                    rebootRouter(backupRouter.getId(), true);
+                    rebootRouter(backupRouter.getId(), true, false);
                 } catch (final ConcurrentOperationException e) {
                     s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e);
                 } catch (final ResourceUnavailableException e) {

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -2502,7 +2502,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
     }
 
     private ConsoleProxyVO stopConsoleProxy(final VMInstanceVO systemVm, final boolean isForced) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
-
         _itMgr.advanceStop(systemVm.getUuid(), isForced);
         return _consoleProxyDao.findById(systemVm.getId());
     }
@@ -2512,6 +2511,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         return _consoleProxyDao.findById(instanceId);
     }
 
+    private ConsoleProxyVO forceRebootConsoleProxy(final VMInstanceVO systemVm)  throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+        _itMgr.advanceStop(systemVm.getUuid(), false);
+        return _consoleProxyMgr.startProxy(systemVm.getId(), true);
+    }
+
     protected ConsoleProxyVO destroyConsoleProxy(final long instanceId) {
         final ConsoleProxyVO proxy = _consoleProxyDao.findById(instanceId);
 
@@ -3401,6 +3405,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         return _secStorageVmDao.findById(instanceId);
     }
 
+    private SecondaryStorageVmVO forceRebootSecondaryStorageVm(final VMInstanceVO systemVm)  throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+        _itMgr.advanceStop(systemVm.getUuid(), false);
+        return _secStorageVmMgr.startSecStorageVm(systemVm.getId());
+    }
+
     protected SecondaryStorageVmVO destroySecondaryStorageVm(final long instanceId) {
         final SecondaryStorageVmVO secStorageVm = _secStorageVmDao.findById(instanceId);
         cleanupDownloadUrlsInZone(secStorageVm.getDataCenterId());
@@ -3570,12 +3579,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             throw ex;
         }
 
-        if (systemVm.getType().equals(VirtualMachine.Type.ConsoleProxy)) {
+        try {
-            ActionEventUtils.startNestedActionEvent(EventTypes.EVENT_PROXY_REBOOT, "rebooting console proxy Vm");
+            if (systemVm.getType().equals(VirtualMachine.Type.ConsoleProxy)) {
-            return rebootConsoleProxy(cmd.getId());
+                ActionEventUtils.startNestedActionEvent(EventTypes.EVENT_PROXY_REBOOT, "rebooting console proxy Vm");
-        } else {
+                if (cmd.isForced()) {
-            ActionEventUtils.startNestedActionEvent(EventTypes.EVENT_SSVM_REBOOT, "rebooting secondary storage Vm");
+                    return forceRebootConsoleProxy(systemVm);
-            return rebootSecondaryStorageVm(cmd.getId());
+                }
+                return rebootConsoleProxy(cmd.getId());
+            } else {
+                ActionEventUtils.startNestedActionEvent(EventTypes.EVENT_SSVM_REBOOT, "rebooting secondary storage Vm");
+                if (cmd.isForced()) {
+                    return forceRebootSecondaryStorageVm(systemVm);
+                }
+                return rebootSecondaryStorageVm(cmd.getId());
+            }
+        } catch (final ResourceUnavailableException e) {
+            throw new CloudRuntimeException("Unable to reboot " + systemVm, e);
+        } catch (final OperationTimedoutException e) {
+            throw new CloudRuntimeException("Operation timed out - Unable to reboot " + systemVm, e);
         }
     }
 

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -800,7 +800,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                     return true;
                 }
 
-                if (rebootVirtualMachine(userId, vmId, false) == null) {
+                if (rebootVirtualMachine(userId, vmId, false, false) == null) {
                     s_logger.warn("Failed to reboot the vm " + vmInstance);
                     return false;
                 } else {
@@ -911,7 +911,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 s_logger.debug("Vm " + vmInstance + " is stopped, not rebooting it as a part of SSH Key reset");
                 return true;
             }
-            if (rebootVirtualMachine(userId, vmId, false) == null) {
+            if (rebootVirtualMachine(userId, vmId, false, false) == null) {
                 s_logger.warn("Failed to reboot the vm " + vmInstance);
                 return false;
             } else {
@@ -948,7 +948,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         return status;
     }
 
-    private UserVm rebootVirtualMachine(long userId, long vmId, boolean enterSetup) throws InsufficientCapacityException, ResourceUnavailableException {
+    private UserVm rebootVirtualMachine(long userId, long vmId, boolean enterSetup, boolean forced) throws InsufficientCapacityException, ResourceUnavailableException {
         UserVmVO vm = _vmDao.findById(vmId);
 
         if (s_logger.isTraceEnabled()) {
@@ -963,6 +963,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (vm.getState() == State.Running && vm.getHostId() != null) {
             collectVmDiskStatistics(vm);
             collectVmNetworkStatistics(vm);
+
+            if (forced) {
+                Host vmOnHost = _hostDao.findById(vm.getHostId());
+                if (vmOnHost == null || vmOnHost.getResourceState() != ResourceState.Enabled || vmOnHost.getStatus() != Status.Up ) {
+                    throw new CloudRuntimeException("Unable to force reboot the VM as the host: " + vm.getHostId() + " is not in the right state");
+                }
+                return forceRebootVirtualMachine(vmId, vm.getHostId(), enterSetup);
+            }
+
             DataCenterVO dc = _dcDao.findById(vm.getDataCenterId());
             try {
                 if (dc.getNetworkType() == DataCenter.NetworkType.Advanced) {
@@ -986,7 +995,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 throw new CloudRuntimeException("Concurrent operations on starting router. " + e);
             } catch (Exception ex){
                 throw new CloudRuntimeException("Router start failed due to" + ex);
-            }finally {
+            } finally {
                 if (s_logger.isInfoEnabled()) {
                     s_logger.info(String.format("Rebooting vm %s%s.", vm.getInstanceName(), enterSetup? " entering hardware setup menu" : " as is"));
                 }
@@ -1007,6 +1016,24 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         }
     }
 
+    private UserVm forceRebootVirtualMachine(long vmId, long hostId, boolean enterSetup) {
+        try {
+            if (stopVirtualMachine(vmId, false) != null) {
+                Map<VirtualMachineProfile.Param,Object> params = null;
+                if (enterSetup) {
+                    params = new HashMap();
+                    params.put(VirtualMachineProfile.Param.BootIntoSetup, Boolean.TRUE);
+                }
+                return startVirtualMachine(vmId, null, null, hostId, params, null).first();
+            }
+        } catch (ResourceUnavailableException e) {
+            throw new CloudRuntimeException("Unable to reboot the VM: " + vmId, e);
+        } catch (CloudException e) {
+            throw new CloudRuntimeException("Unable to reboot the VM: " + vmId, e);
+        }
+        return null;
+    }
+
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_VM_UPGRADE, eventDescription = "upgrading Vm")
     /*
@@ -2894,7 +2921,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         // Verify input parameters
         UserVmVO vmInstance = _vmDao.findById(vmId);
         if (vmInstance == null) {
-            throw new InvalidParameterValueException("unable to find a virtual machine with id " + vmId);
+            throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
         }
 
         _accountMgr.checkAccess(caller, null, true, vmInstance);
@@ -2916,7 +2943,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (enterSetup != null && enterSetup && !HypervisorType.VMware.equals(vmInstance.getHypervisorType())) {
             throw new InvalidParameterValueException("Booting into a hardware setup menu is not implemented on " + vmInstance.getHypervisorType());
         }
-        UserVm userVm = rebootVirtualMachine(CallContext.current().getCallingUserId(), vmId, enterSetup == null ? false : cmd.getBootIntoSetup());
+
+        UserVm userVm = rebootVirtualMachine(CallContext.current().getCallingUserId(), vmId, enterSetup == null ? false : cmd.getBootIntoSetup(), cmd.isForced());
         if (userVm != null ) {
             // update the vmIdCountMap if the vm is in advanced shared network with out services
             final List<NicVO> nics = _nicDao.listByVmId(vmId);

server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java

@@ -110,7 +110,7 @@ public class MockVpcVirtualNetworkApplianceManager extends ManagerBase implement
      * @see com.cloud.network.VirtualNetworkApplianceService#rebootRouter(long, boolean)
      */
     @Override
-    public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException {
+    public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork, final boolean forced) throws ConcurrentOperationException, ResourceUnavailableException {
         // TODO Auto-generated method stub
         return null;
     }

test/integration/smoke/test_routers.py

@@ -815,3 +815,46 @@ class TestRouterServices(cloudstackTestCase):
             "Router response after reboot is either is invalid\
                     or in stopped state")
         return
+
+    @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false")
+    def test_10_reboot_router_forced(self):
+        """Test force reboot router
+        """
+
+        list_router_response = list_routers(
+            self.apiclient,
+            account=self.account.name,
+            domainid=self.account.domainid
+        )
+        self.assertEqual(
+            isinstance(list_router_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+        router = list_router_response[0]
+
+        public_ip = router.publicip
+
+        self.debug("Force rebooting the router with ID: %s" % router.id)
+        # Reboot the router
+        cmd = rebootRouter.rebootRouterCmd()
+        cmd.id = router.id
+        cmd.forced = True
+        self.apiclient.rebootRouter(cmd)
+
+        # List routers to check state of router
+        retries_cnt = 10
+        while retries_cnt >= 0:
+            router_response = list_routers(
+                self.apiclient,
+                id=router.id
+            )
+            if self.verifyRouterResponse(router_response, public_ip):
+                self.debug("Router is running successfully after force reboot")
+                return
+            time.sleep(10)
+            retries_cnt = retries_cnt - 1
+        self.fail(
+            "Router response after force reboot is either invalid\
+                    or router in stopped state")
+        return

test/integration/smoke/test_ssvm.py

@@ -959,7 +959,122 @@ class TestSSVMs(cloudstackTestCase):
             "basic",
             "sg"],
         required_hardware="true")
-    def test_09_destroy_ssvm(self):
+    def test_09_reboot_ssvm_forced(self):
+        """Test force reboot SSVM
+        """
+
+        list_ssvm_response = list_ssvms(
+            self.apiclient,
+            systemvmtype='secondarystoragevm',
+            state='Running',
+            zoneid=self.zone.id
+        )
+
+        self.assertEqual(
+            isinstance(list_ssvm_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        ssvm_response = list_ssvm_response[0]
+
+        hosts = list_hosts(
+            self.apiclient,
+            id=ssvm_response.hostid
+        )
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.debug("Force rebooting SSVM: %s" % ssvm_response.id)
+        cmd = rebootSystemVm.rebootSystemVmCmd()
+        cmd.id = ssvm_response.id
+        cmd.forced = True
+        self.apiclient.rebootSystemVm(cmd)
+
+        ssvm_response = self.checkForRunningSystemVM(ssvm_response)
+        self.debug("SSVM State: %s" % ssvm_response.state)
+        self.assertEqual(
+            'Running',
+            str(ssvm_response.state),
+            "Check whether SSVM is running or not"
+        )
+
+        # Wait for the agent to be up
+        self.waitForSystemVMAgent(ssvm_response.name)
+
+        # Wait until NFS stores mounted before running the script
+        time.sleep(90)
+        # Call to verify cloud process is running
+        self.test_03_ssvm_internals()
+
+    @attr(
+        tags=[
+            "advanced",
+            "advancedns",
+            "smoke",
+            "basic",
+            "sg"],
+        required_hardware="true")
+    def test_10_reboot_cpvm_forced(self):
+        """Test force reboot CPVM
+        """
+
+        list_cpvm_response = list_ssvms(
+            self.apiclient,
+            systemvmtype='consoleproxy',
+            state='Running',
+            zoneid=self.zone.id
+        )
+        self.assertEqual(
+            isinstance(list_cpvm_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+        cpvm_response = list_cpvm_response[0]
+
+        hosts = list_hosts(
+            self.apiclient,
+            id=cpvm_response.hostid
+        )
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.debug("Force rebooting CPVM: %s" % cpvm_response.id)
+
+        cmd = rebootSystemVm.rebootSystemVmCmd()
+        cmd.id = cpvm_response.id
+        cmd.forced = True
+        self.apiclient.rebootSystemVm(cmd)
+
+        cpvm_response = self.checkForRunningSystemVM(cpvm_response)
+        self.debug("CPVM state: %s" % cpvm_response.state)
+        self.assertEqual(
+            'Running',
+            str(cpvm_response.state),
+            "Check whether CPVM is running or not"
+        )
+
+        # Wait for the agent to be up
+        self.waitForSystemVMAgent(cpvm_response.name)
+
+        # Call to verify cloud process is running
+        self.test_04_cpvm_internals()
+
+    @attr(
+        tags=[
+            "advanced",
+            "advancedns",
+            "smoke",
+            "basic",
+            "sg"],
+        required_hardware="true")
+    def test_11_destroy_ssvm(self):
         """Test destroy SSVM
         """
 
@@ -1031,7 +1146,7 @@ class TestSSVMs(cloudstackTestCase):
             "basic",
             "sg"],
         required_hardware="true")
-    def test_10_destroy_cpvm(self):
+    def test_12_destroy_cpvm(self):
         """Test destroy CPVM
         """
 
@@ -1102,7 +1217,7 @@ class TestSSVMs(cloudstackTestCase):
             "basic",
             "sg"],
         required_hardware="true")
-    def test_11_ss_nfs_version_on_ssvm(self):
+    def test_13_ss_nfs_version_on_ssvm(self):
         """Test NFS Version on Secondary Storage mounted properly on SSVM
         """
 

test/integration/smoke/test_vm_life_cycle.py

@@ -499,6 +499,40 @@ class TestVMLifeCycle(cloudstackTestCase):
         )
         return
 
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_04_reboot_vm_forced(self):
+        """Test Force Reboot Virtual Machine
+        """
+
+        try:
+            self.debug("Force rebooting VM - ID: %s" % self.virtual_machine.id)
+            self.small_virtual_machine.reboot(self.apiclient, forced=True)
+        except Exception as e:
+            self.fail("Failed to force reboot VM: %s" % e)
+
+        list_vm_response = VirtualMachine.list(
+            self.apiclient,
+            id=self.small_virtual_machine.id
+        )
+        self.assertEqual(
+            isinstance(list_vm_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertNotEqual(
+            len(list_vm_response),
+            0,
+            "Check VM available in List Virtual Machines"
+        )
+
+        self.assertEqual(
+            list_vm_response[0].state,
+            "Running",
+            "Check virtual machine is in running state"
+        )
+        return
+
     @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
     def test_06_destroy_vm(self):
         """Test destroy Virtual Machine

tools/marvin/marvin/lib/base.py

@@ -723,10 +723,12 @@ class VirtualMachine:
             raise Exception(response[1])
         return
 
-    def reboot(self, apiclient):
+    def reboot(self, apiclient, forced=None):
         """Reboot the instance"""
         cmd = rebootVirtualMachine.rebootVirtualMachineCmd()
         cmd.id = self.id
+        if forced:
+            cmd.forced = forced
         apiclient.rebootVirtualMachine(cmd)
 
         response = self.getState(apiclient, VirtualMachine.RUNNING)
@@ -4448,10 +4450,12 @@ class Router:
         return apiclient.stopRouter(cmd)
 
     @classmethod
-    def reboot(cls, apiclient, id):
+    def reboot(cls, apiclient, id, forced=None):
         """Reboots the router"""
         cmd = rebootRouter.rebootRouterCmd()
         cmd.id = id
+        if forced:
+            cmd.forced = forced
         return apiclient.rebootRouter(cmd)
 
     @classmethod

ui/src/config/section/compute.js

@@ -125,6 +125,7 @@ export default {
           show: (record) => { return ['Running'].includes(record.state) },
           args: (record, store) => {
             var fields = []
+            fields.push('forced')
             if (record.hypervisor === 'VMware') {
               if (store.apis.rebootVirtualMachine.params.filter(x => x.name === 'bootintosetup').length > 0) {
                 fields.push('bootintosetup')

ui/src/config/section/infra/routers.js

@@ -66,6 +66,7 @@ export default {
       label: 'label.action.reboot.router',
       message: 'message.action.reboot.router',
       dataView: true,
+      args: ['forced'],
       hidden: (record) => { return record.state === 'Running' }
     },
     {

ui/src/config/section/infra/systemVms.js

@@ -47,7 +47,8 @@ export default {
       label: 'label.action.reboot.systemvm',
       message: 'message.action.reboot.systemvm',
       dataView: true,
-      show: (record) => { return record.state === 'Running' }
+      show: (record) => { return record.state === 'Running' },
+      args: ['forced']
     },
     {
       api: 'scaleSystemVm',