From 81dfcbb5f56b616b9edcf97fffedb26c187a7fc2 Mon Sep 17 00:00:00 2001 From: sureshanaparti <12028987+sureshanaparti@users.noreply.github.com> Date: Sat, 6 Mar 2021 14:58:56 +0530 Subject: [PATCH] server: Use ACPI event to reboot VM on KVM, and Use 'forced' reboot option to stop and start the VM(s) (#4681) * Updated libvirt's native reboot operation for VM on KVM using ACPI event, and Added 'forced' reboot option to stop and start the VM (using rebootVirtualMachine API) * Added 'forced' reboot option for System VM and Router - New parameter 'forced' in rebootSystemVm API, to stop and then start System VM - New parameter 'forced' in rebootRouter API, to force stop and then start Router * Added force reboot tests for User VM, System VM and Router --- .../VirtualNetworkApplianceService.java | 2 +- .../command/admin/router/RebootRouterCmd.java | 9 +- .../admin/systemvm/RebootSystemVmCmd.java | 7 + .../api/command/user/vm/RebootVMCmd.java | 7 + .../cloud/vm/VirtualMachineManagerImpl.java | 6 +- .../resource/LibvirtComputingResource.java | 31 +---- .../VirtualNetworkApplianceManagerImpl.java | 6 +- .../cloud/server/ManagementServerImpl.java | 35 ++++- .../java/com/cloud/vm/UserVmManagerImpl.java | 40 +++++- ...MockVpcVirtualNetworkApplianceManager.java | 2 +- test/integration/smoke/test_routers.py | 43 +++++++ test/integration/smoke/test_ssvm.py | 121 +++++++++++++++++- test/integration/smoke/test_vm_life_cycle.py | 34 +++++ tools/marvin/marvin/lib/base.py | 8 +- ui/src/config/section/compute.js | 1 + ui/src/config/section/infra/routers.js | 1 + ui/src/config/section/infra/systemVms.js | 3 +- 17 files changed, 303 insertions(+), 53 deletions(-) diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index 8504efda509..92a664f7f95 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -44,7 +44,7 @@ public interface VirtualNetworkApplianceService { * the command specifying router's id * @return router if successful */ - VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; + VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork, boolean forced) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; VirtualRouter upgradeRouter(UpgradeRouterCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/RebootRouterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/RebootRouterCmd.java index 802e3df3dcf..764d304e25b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/RebootRouterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/RebootRouterCmd.java @@ -49,6 +49,9 @@ public class RebootRouterCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, required = true, description = "the ID of the router") private Long id; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the router (Router is force Stopped and then Started)", since = "4.16.0") + private Boolean forced; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -96,10 +99,14 @@ public class RebootRouterCmd extends BaseAsyncCmd { return getId(); } + public boolean isForced() { + return (forced != null) ? forced : false; + } + @Override public void execute() throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { CallContext.current().setEventDetails("Router Id: " + this._uuidMgr.getUuid(VirtualMachine.class,getId())); - VirtualRouter result = _routerService.rebootRouter(getId(), true); + VirtualRouter result = _routerService.rebootRouter(getId(), true, isForced()); if (result != null) { DomainRouterResponse response = _responseGenerator.createDomainRouterResponse(result); response.setResponseName("router"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/RebootSystemVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/RebootSystemVmCmd.java index ebc50ae7e1d..352fd3b330b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/RebootSystemVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/RebootSystemVmCmd.java @@ -52,6 +52,9 @@ public class RebootSystemVmCmd extends BaseAsyncCmd { description = "The ID of the system virtual machine") private Long id; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the system VM (System VM is Stopped and then Started)", since = "4.16.0") + private Boolean forced; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -104,6 +107,10 @@ public class RebootSystemVmCmd extends BaseAsyncCmd { return getId(); } + public boolean isForced() { + return (forced != null) ? forced : false; + } + @Override public void execute() { CallContext.current().setEventDetails("Vm Id: " + this._uuidMgr.getUuid(VirtualMachine.class, getId())); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java index 5bdbbb651eb..d827a6b894a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java @@ -53,6 +53,9 @@ public class RebootVMCmd extends BaseAsyncCmd implements UserCmd { required=true, description="The ID of the virtual machine") private Long id; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force reboot the VM (VM is Stopped and then Started)", since = "4.16.0") + private Boolean forced; + @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup menu or not", since = "4.15.0.0") private Boolean bootIntoSetup; @@ -64,6 +67,10 @@ public class RebootVMCmd extends BaseAsyncCmd implements UserCmd { return id; } + public boolean isForced() { + return (forced != null) ? forced : false; + } + public Boolean getBootIntoSetup() { return bootIntoSetup; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index dfec0b1de77..2bbb8cbe7c5 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3458,7 +3458,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final DeployDestination dest = new DeployDestination(dc, pod, cluster, host); try { - final Commands cmds = new Commands(Command.OnError.Stop); RebootCommand rebootCmd = new RebootCommand(vm.getInstanceName(), getExecuteInSequence(vm.getHypervisorType())); VirtualMachineTO vmTo = getVmTO(vm.getId()); @@ -3476,7 +3475,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } return; } - s_logger.info("Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? " no reboot answer" : rebootAnswer.getDetails())); + + String errorMsg = "Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? "no reboot response" : rebootAnswer.getDetails()); + s_logger.info(errorMsg); + throw new CloudRuntimeException(errorMsg); } catch (final OperationTimedoutException e) { s_logger.warn("Unable to send the reboot command to host " + dest.getHost() + " for the vm " + vm + " due to operation timeout", e); throw new CloudRuntimeException("Failed to reboot the vm on host " + dest.getHost()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index cfa5474e61e..e380ef187f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -134,7 +134,6 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.GuestDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.GuestResourceDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InputDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; -import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.GuestNetType; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef.RngBackendModel; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SCSIDef; @@ -3228,35 +3227,15 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv String msg = null; try { dm = conn.domainLookupByName(vmName); - // Get XML Dump including the secure information such as VNC password - // By passing 1, or VIR_DOMAIN_XML_SECURE flag - // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainXMLFlags - String vmDef = dm.getXMLDesc(1); - final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); - parser.parseDomainXML(vmDef); - for (final InterfaceDef nic : parser.getInterfaces()) { - if (nic.getNetType() == GuestNetType.BRIDGE && nic.getBrName().startsWith("cloudVirBr")) { - try { - final int vnetId = Integer.parseInt(nic.getBrName().replaceFirst("cloudVirBr", "")); - final String pifName = getPif(_guestBridgeName); - final String newBrName = "br" + pifName + "-" + vnetId; - vmDef = vmDef.replace("'" + nic.getBrName() + "'", "'" + newBrName + "'"); - s_logger.debug("VM bridge name is changed from " + nic.getBrName() + " to " + newBrName); - } catch (final NumberFormatException e) { - continue; - } - } - } - s_logger.debug(vmDef); - msg = stopVM(conn, vmName, false); - msg = startVM(conn, vmName, vmDef); + // Perform ACPI based reboot + // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainReboot + // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainRebootFlagValues + // Send ACPI event to Reboot + dm.reboot(0x1); return null; } catch (final LibvirtException e) { s_logger.warn("Failed to create vm", e); msg = e.getMessage(); - } catch (final InternalErrorException e) { - s_logger.warn("Failed to create vm", e); - msg = e.getMessage(); } finally { try { if (dm != null) { diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 8e38fb9280b..2657b3f1812 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -510,7 +510,7 @@ Configurable, StateListener params = null; + if (enterSetup) { + params = new HashMap(); + params.put(VirtualMachineProfile.Param.BootIntoSetup, Boolean.TRUE); + } + return startVirtualMachine(vmId, null, null, hostId, params, null).first(); + } + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException("Unable to reboot the VM: " + vmId, e); + } catch (CloudException e) { + throw new CloudRuntimeException("Unable to reboot the VM: " + vmId, e); + } + return null; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_UPGRADE, eventDescription = "upgrading Vm") /* @@ -2894,7 +2921,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // Verify input parameters UserVmVO vmInstance = _vmDao.findById(vmId); if (vmInstance == null) { - throw new InvalidParameterValueException("unable to find a virtual machine with id " + vmId); + throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); } _accountMgr.checkAccess(caller, null, true, vmInstance); @@ -2916,7 +2943,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (enterSetup != null && enterSetup && !HypervisorType.VMware.equals(vmInstance.getHypervisorType())) { throw new InvalidParameterValueException("Booting into a hardware setup menu is not implemented on " + vmInstance.getHypervisorType()); } - UserVm userVm = rebootVirtualMachine(CallContext.current().getCallingUserId(), vmId, enterSetup == null ? false : cmd.getBootIntoSetup()); + + UserVm userVm = rebootVirtualMachine(CallContext.current().getCallingUserId(), vmId, enterSetup == null ? false : cmd.getBootIntoSetup(), cmd.isForced()); if (userVm != null ) { // update the vmIdCountMap if the vm is in advanced shared network with out services final List nics = _nicDao.listByVmId(vmId); diff --git a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java index abb1863a1a3..e734fdfc8ad 100644 --- a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java +++ b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java @@ -110,7 +110,7 @@ public class MockVpcVirtualNetworkApplianceManager extends ManagerBase implement * @see com.cloud.network.VirtualNetworkApplianceService#rebootRouter(long, boolean) */ @Override - public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException { + public VirtualRouter rebootRouter(final long routerId, final boolean reprogramNetwork, final boolean forced) throws ConcurrentOperationException, ResourceUnavailableException { // TODO Auto-generated method stub return null; } diff --git a/test/integration/smoke/test_routers.py b/test/integration/smoke/test_routers.py index 3a20d64fe15..356bd213cfc 100644 --- a/test/integration/smoke/test_routers.py +++ b/test/integration/smoke/test_routers.py @@ -815,3 +815,46 @@ class TestRouterServices(cloudstackTestCase): "Router response after reboot is either is invalid\ or in stopped state") return + + @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false") + def test_10_reboot_router_forced(self): + """Test force reboot router + """ + + list_router_response = list_routers( + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) + self.assertEqual( + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) + router = list_router_response[0] + + public_ip = router.publicip + + self.debug("Force rebooting the router with ID: %s" % router.id) + # Reboot the router + cmd = rebootRouter.rebootRouterCmd() + cmd.id = router.id + cmd.forced = True + self.apiclient.rebootRouter(cmd) + + # List routers to check state of router + retries_cnt = 10 + while retries_cnt >= 0: + router_response = list_routers( + self.apiclient, + id=router.id + ) + if self.verifyRouterResponse(router_response, public_ip): + self.debug("Router is running successfully after force reboot") + return + time.sleep(10) + retries_cnt = retries_cnt - 1 + self.fail( + "Router response after force reboot is either invalid\ + or router in stopped state") + return diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index bb83931c1fd..0392d9eb9f1 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -959,7 +959,122 @@ class TestSSVMs(cloudstackTestCase): "basic", "sg"], required_hardware="true") - def test_09_destroy_ssvm(self): + def test_09_reboot_ssvm_forced(self): + """Test force reboot SSVM + """ + + list_ssvm_response = list_ssvms( + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + zoneid=self.zone.id + ) + + self.assertEqual( + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) + + ssvm_response = list_ssvm_response[0] + + hosts = list_hosts( + self.apiclient, + id=ssvm_response.hostid + ) + self.assertEqual( + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) + + self.debug("Force rebooting SSVM: %s" % ssvm_response.id) + cmd = rebootSystemVm.rebootSystemVmCmd() + cmd.id = ssvm_response.id + cmd.forced = True + self.apiclient.rebootSystemVm(cmd) + + ssvm_response = self.checkForRunningSystemVM(ssvm_response) + self.debug("SSVM State: %s" % ssvm_response.state) + self.assertEqual( + 'Running', + str(ssvm_response.state), + "Check whether SSVM is running or not" + ) + + # Wait for the agent to be up + self.waitForSystemVMAgent(ssvm_response.name) + + # Wait until NFS stores mounted before running the script + time.sleep(90) + # Call to verify cloud process is running + self.test_03_ssvm_internals() + + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") + def test_10_reboot_cpvm_forced(self): + """Test force reboot CPVM + """ + + list_cpvm_response = list_ssvms( + self.apiclient, + systemvmtype='consoleproxy', + state='Running', + zoneid=self.zone.id + ) + self.assertEqual( + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) + cpvm_response = list_cpvm_response[0] + + hosts = list_hosts( + self.apiclient, + id=cpvm_response.hostid + ) + self.assertEqual( + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) + + self.debug("Force rebooting CPVM: %s" % cpvm_response.id) + + cmd = rebootSystemVm.rebootSystemVmCmd() + cmd.id = cpvm_response.id + cmd.forced = True + self.apiclient.rebootSystemVm(cmd) + + cpvm_response = self.checkForRunningSystemVM(cpvm_response) + self.debug("CPVM state: %s" % cpvm_response.state) + self.assertEqual( + 'Running', + str(cpvm_response.state), + "Check whether CPVM is running or not" + ) + + # Wait for the agent to be up + self.waitForSystemVMAgent(cpvm_response.name) + + # Call to verify cloud process is running + self.test_04_cpvm_internals() + + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") + def test_11_destroy_ssvm(self): """Test destroy SSVM """ @@ -1031,7 +1146,7 @@ class TestSSVMs(cloudstackTestCase): "basic", "sg"], required_hardware="true") - def test_10_destroy_cpvm(self): + def test_12_destroy_cpvm(self): """Test destroy CPVM """ @@ -1102,7 +1217,7 @@ class TestSSVMs(cloudstackTestCase): "basic", "sg"], required_hardware="true") - def test_11_ss_nfs_version_on_ssvm(self): + def test_13_ss_nfs_version_on_ssvm(self): """Test NFS Version on Secondary Storage mounted properly on SSVM """ diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 08668e47b72..a64293be297 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -499,6 +499,40 @@ class TestVMLifeCycle(cloudstackTestCase): ) return + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_reboot_vm_forced(self): + """Test Force Reboot Virtual Machine + """ + + try: + self.debug("Force rebooting VM - ID: %s" % self.virtual_machine.id) + self.small_virtual_machine.reboot(self.apiclient, forced=True) + except Exception as e: + self.fail("Failed to force reboot VM: %s" % e) + + list_vm_response = VirtualMachine.list( + self.apiclient, + id=self.small_virtual_machine.id + ) + self.assertEqual( + isinstance(list_vm_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_vm_response), + 0, + "Check VM available in List Virtual Machines" + ) + + self.assertEqual( + list_vm_response[0].state, + "Running", + "Check virtual machine is in running state" + ) + return + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_06_destroy_vm(self): """Test destroy Virtual Machine diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 20e116ae1bc..a05f18b824e 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -723,10 +723,12 @@ class VirtualMachine: raise Exception(response[1]) return - def reboot(self, apiclient): + def reboot(self, apiclient, forced=None): """Reboot the instance""" cmd = rebootVirtualMachine.rebootVirtualMachineCmd() cmd.id = self.id + if forced: + cmd.forced = forced apiclient.rebootVirtualMachine(cmd) response = self.getState(apiclient, VirtualMachine.RUNNING) @@ -4448,10 +4450,12 @@ class Router: return apiclient.stopRouter(cmd) @classmethod - def reboot(cls, apiclient, id): + def reboot(cls, apiclient, id, forced=None): """Reboots the router""" cmd = rebootRouter.rebootRouterCmd() cmd.id = id + if forced: + cmd.forced = forced return apiclient.rebootRouter(cmd) @classmethod diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 6acebf6e8cf..c3d2623885d 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -125,6 +125,7 @@ export default { show: (record) => { return ['Running'].includes(record.state) }, args: (record, store) => { var fields = [] + fields.push('forced') if (record.hypervisor === 'VMware') { if (store.apis.rebootVirtualMachine.params.filter(x => x.name === 'bootintosetup').length > 0) { fields.push('bootintosetup') diff --git a/ui/src/config/section/infra/routers.js b/ui/src/config/section/infra/routers.js index 6ffa4680a80..a338237c5fb 100644 --- a/ui/src/config/section/infra/routers.js +++ b/ui/src/config/section/infra/routers.js @@ -66,6 +66,7 @@ export default { label: 'label.action.reboot.router', message: 'message.action.reboot.router', dataView: true, + args: ['forced'], hidden: (record) => { return record.state === 'Running' } }, { diff --git a/ui/src/config/section/infra/systemVms.js b/ui/src/config/section/infra/systemVms.js index bc20b904d6c..5c02734e34f 100644 --- a/ui/src/config/section/infra/systemVms.js +++ b/ui/src/config/section/infra/systemVms.js @@ -47,7 +47,8 @@ export default { label: 'label.action.reboot.systemvm', message: 'message.action.reboot.systemvm', dataView: true, - show: (record) => { return record.state === 'Running' } + show: (record) => { return record.state === 'Running' }, + args: ['forced'] }, { api: 'scaleSystemVm',