Merge remote-tracking branch 'origin/4.9'

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

api/src/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java

@@ -54,7 +54,7 @@ public class StopVMCmd extends BaseAsyncCmd {
     private Long id;
 
     @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM "
-        + "(vm is marked as Stopped even when command fails to be send to the backend).  The caller knows the VM is stopped.")
+        + "(vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted).  The caller knows the VM is stopped.")
     private Boolean forced;
 
     // ///////////////////////////////////////////////////

core/src/com/cloud/agent/api/StopCommand.java

@@ -29,6 +29,7 @@ public class StopCommand extends RebootCommand {
     private GPUDeviceTO gpuDevice;
     boolean checkBeforeCleanup = false;
     String controlIp = null;
+    boolean forceStop = false;
 
     protected StopCommand() {
     }
@@ -46,6 +47,12 @@ public class StopCommand extends RebootCommand {
         this.checkBeforeCleanup = checkBeforeCleanup;
     }
 
+    public StopCommand(VirtualMachine vm, boolean executeInSequence, boolean checkBeforeCleanup, boolean forceStop) {
+        super(vm.getInstanceName(), executeInSequence);
+        this.checkBeforeCleanup = checkBeforeCleanup;
+        this.forceStop = forceStop;
+    }
+
     public StopCommand(String vmName, boolean executeInSequence, boolean checkBeforeCleanup) {
         super(vmName, executeInSequence);
         this.checkBeforeCleanup = checkBeforeCleanup;
@@ -89,6 +96,10 @@ public class StopCommand extends RebootCommand {
     }
 
     public void setControlIp(String controlIp){
-        this.controlIp =controlIp;
+        this.controlIp = controlIp;
+    }
+
+    public boolean isForceStop() {
+        return forceStop;
     }
 }

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1551,9 +1551,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
 
         vmGuru.prepareStop(profile);
-        StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+
-        stpCmd.setControlIp(getControlNicIpForVM(vm));
+        final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false, cleanUpEvenIfUnableToStop);
-        final StopCommand stop = stpCmd;
+        stop.setControlIp(getControlNicIpForVM(vm));
 
         boolean stopped = false;
         Answer answer = null;

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2799,7 +2799,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
                 }
             }
             s_logger.debug(vmDef);
-            msg = stopVM(conn, vmName);
+            msg = stopVM(conn, vmName, false);
             msg = startVM(conn, vmName, vmDef);
             return null;
         } catch (final LibvirtException e) {
@@ -2821,7 +2821,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return msg;
     }
 
-    public String stopVM(final Connect conn, final String vmName) {
+    public String stopVM(final Connect conn, final String vmName, final boolean forceStop) {
         DomainState state = null;
         Domain dm = null;
 
@@ -2842,9 +2842,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         }
 
         s_logger.debug("Try to stop the vm at first");
-        String ret = stopVM(conn, vmName, false);
+        if (forceStop) {
+            return stopVMInternal(conn, vmName, true);
+        }
+        String ret = stopVMInternal(conn, vmName, false);
         if (ret == Script.ERR_TIMEOUT) {
-            ret = stopVM(conn, vmName, true);
+            ret = stopVMInternal(conn, vmName, true);
         } else if (ret != null) {
             /*
              * There is a race condition between libvirt and qemu: libvirt
@@ -2877,7 +2880,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
             if (state != DomainState.VIR_DOMAIN_SHUTOFF) {
                 s_logger.debug("Try to destroy the vm");
-                ret = stopVM(conn, vmName, true);
+                ret = stopVMInternal(conn, vmName, true);
                 if (ret != null) {
                     return ret;
                 }
@@ -2887,7 +2890,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return null;
     }
 
-    protected String stopVM(final Connect conn, final String vmName, final boolean force) {
+    protected String stopVMInternal(final Connect conn, final String vmName, final boolean force) {
         Domain dm = null;
         try {
             dm = conn.domainLookupByName(vmName);

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java

@@ -85,7 +85,8 @@ public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand,
             final List<InterfaceDef> ifaces = libvirtComputingResource.getInterfaces(conn, vmName);
 
             libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
-            final String result = libvirtComputingResource.stopVM(conn, vmName);
+            final String result = libvirtComputingResource.stopVM(conn, vmName, command.isForceStop());
+
             if (result == null) {
                 for (final DiskDef disk : disks) {
                     libvirtComputingResource.cleanupDisk(disk);

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -3238,13 +3238,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
                     vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_VM_INTERNAL_NAME, cmd.getVmName());
 
                     if (getVmPowerState(vmMo) != PowerState.PowerOff) {
-                        if (vmMo.safePowerOff(_shutdownWaitMs)) {
+                        String msg = "Stop VM " + cmd.getVmName() + " Succeed";
-                            return new StopAnswer(cmd, "Stop VM " + cmd.getVmName() + " Succeed", true);
+                        boolean success = false;
+                        if (cmd.isForceStop()) {
+                            success = vmMo.powerOff();
                         } else {
-                            String msg = "Have problem in powering off VM " + cmd.getVmName() + ", let the process continue";
+                            success = vmMo.safePowerOff(_shutdownWaitMs);
-                            s_logger.warn(msg);
-                            return new StopAnswer(cmd, msg, true);
                         }
+                        if (!success) {
+                            msg = "Have problem in powering off VM " + cmd.getVmName() + ", let the process continue";
+                            s_logger.warn(msg);
+                        }
+                        return new StopAnswer(cmd, msg, true);
                     }
 
                     String msg = "VM " + cmd.getVmName() + " is already in stopped state";

plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -4931,10 +4931,15 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         return null;
     }
 
-    public void shutdownVM(final Connection conn, final VM vm, final String vmName) throws XmlRpcException {
+    public void shutdownVM(final Connection conn, final VM vm, final String vmName, final boolean forcedStop) throws XmlRpcException {
         Task task = null;
         try {
-            task = vm.cleanShutdownAsync(conn);
+            if (forcedStop) {
+                task = vm.hardShutdownAsync(conn);
+            } else {
+                task = vm.cleanShutdownAsync(conn);
+            }
+
             try {
                 // poll every 1 seconds , timeout after 10 minutes
                 waitForTask(conn, task, 1000, 10 * 60 * 1000);
@@ -4947,7 +4952,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 throw new CloudRuntimeException("Shutdown VM catch HandleInvalid and VM is not in HALTED state");
             }
         } catch (final XenAPIException e) {
-            s_logger.debug("Unable to cleanShutdown VM(" + vmName + ") on host(" + _host.getUuid() + ") due to " + e.toString());
+            s_logger.debug("Unable to shutdown VM(" + vmName + ") with force=" + forcedStop + " on host(" + _host.getUuid() + ") due to " + e.toString());
             try {
                 VmPowerState state = vm.getPowerState(conn);
                 if (state == VmPowerState.RUNNING) {

plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStopCommandWrapper.java

@@ -113,7 +113,7 @@ public final class CitrixStopCommandWrapper extends CommandWrapper<StopCommand,
                                 s_logger.info("Removed  network rules for vm " + command.getVmName());
                             }
                         }
-                        citrixResourceBase.shutdownVM(conn, vm, vmName);
+                        citrixResourceBase.shutdownVM(conn, vm, vmName, command.isForceStop());
                     }
                 } catch (final Exception e) {
                     final String msg = "Catch exception " + e.getClass().getName() + " when stop VM:" + command.getVmName() + " due to " + e.toString();

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -45,36 +45,6 @@ from cs.CsProcess import CsProcess
 from cs.CsStaticRoutes import CsStaticRoutes
 
 
-class CsPassword(CsDataBag):
-
-    TOKEN_FILE="/tmp/passwdsrvrtoken"
-
-    def process(self):
-        for item in self.dbag:
-            if item == "id":
-                continue
-            self.__update(item, self.dbag[item])
-
-    def __update(self, vm_ip, password):
-        token = ""
-        try:
-            tokenFile = open(self.TOKEN_FILE)
-            token = tokenFile.read()
-        except IOError:
-            logging.debug("File %s does not exist" % self.TOKEN_FILE)
-
-        ips_cmd = "ip addr show | grep inet | awk '{print $2}'"
-        ips = CsHelper.execute(ips_cmd)
-        for ip in ips:
-            server_ip = ip.split('/')[0]
-            proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip])
-            if proc.find():
-                update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \
-                '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token)
-                result = CsHelper.execute(update_command)
-                logging.debug("Update password server result ==> %s" % result)
-
-
 class CsAcl(CsDataBag):
     """
         Deal with Network acls

systemvm/patches/debian/config/opt/cloud/bin/cs/CsVmPassword.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -- coding: utf-8 --
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import CsHelper
+from CsProcess import CsProcess
+from netaddr import IPNetwork, IPAddress
+import logging
+
+
+class CsPassword:
+
+    TOKEN_FILE="/tmp/passwdsrvrtoken"
+
+    def __init__(self, dbag):
+        self.dbag = dbag
+        self.process()
+
+    def process(self):
+        self.__update(self.dbag['ip_address'], self.dbag['password'])
+
+    def __update(self, vm_ip, password):
+        token = ""
+        try:
+            tokenFile = open(self.TOKEN_FILE)
+            token = tokenFile.read()
+        except IOError:
+            logging.debug("File %s does not exist" % self.TOKEN_FILE)
+
+        logging.debug("Got VM '%s' and password '%s'" % (vm_ip, password))
+        get_cidrs_cmd = "ip addr show | grep inet | grep -v secondary | awk '{print $2}'"
+        cidrs = CsHelper.execute(get_cidrs_cmd)
+        logging.debug("Found these CIDRs: %s" % cidrs)
+        for cidr in cidrs:
+            logging.debug("Processing CIDR '%s'" % cidr)
+            if IPAddress(vm_ip) in IPNetwork(cidr):
+                ip = cidr.split('/')[0]
+                logging.debug("Cidr %s matches vm ip address %s so adding passwd to passwd server at %s" % (cidr, vm_ip, ip))
+                proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', ip])
+                if proc.find():
+                    update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \
+                                     '-F "token={TOKEN}" --interface 127.0.0.1 >/dev/null 2>/dev/null &'.format(SERVER_IP=ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token)
+                    result = CsHelper.execute(update_command)
+                    logging.debug("Update password server result ==> %s" % result)
+                else:
+                    logging.debug("Update password server skipped because we didn't find a passwd server process for %s (makes sense on backup routers)" % ip)

systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py

@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pprint import pprint
-from netaddr import *
-
-
-def merge(dbag, data):
-    """
-    Track vm passwords
-    """
-    dbag[data['ip_address']] = data['password']
-    return dbag

systemvm/patches/debian/config/opt/cloud/bin/merge.py

@@ -23,7 +23,6 @@ import logging
 import cs_ip
 import cs_guestnetwork
 import cs_cmdline
-import cs_vmp
 import cs_network_acl
 import cs_firewallrules
 import cs_loadbalancer
@@ -36,8 +35,6 @@ import cs_remoteaccessvpn
 import cs_vpnusers
 import cs_staticroutes
 
-from pprint import pprint
-
 
 class DataBag:
 
@@ -105,8 +102,6 @@ class updateDataBag:
             dbag = self.processGuestNetwork(self.db.getDataBag())
         elif self.qFile.type == 'cmdline':
             dbag = self.processCL(self.db.getDataBag())
-        elif self.qFile.type == 'vmpassword':
-            dbag = self.processVMpassword(self.db.getDataBag())
         elif self.qFile.type == 'networkacl':
             dbag = self.process_network_acl(self.db.getDataBag())
         elif self.qFile.type == 'firewallrules':
@@ -188,9 +183,6 @@ class updateDataBag:
     def process_staticroutes(self, dbag):
         return cs_staticroutes.merge(dbag, self.qFile.data)
 
-    def processVMpassword(self, dbag):
-        return cs_vmp.merge(dbag, self.qFile.data)
-
     def processForwardingRules(self, dbag):
         # to be used by both staticnat and portforwarding
         return cs_forwardingrules.merge(dbag, self.qFile.data)
@@ -275,13 +267,21 @@ class QueueFile:
     fileName = ''
     configCache = "/var/cache/cloud"
     keep = True
+    do_merge = True
     data = {}
 
+    def update_databag(self):
+        if self.do_merge:
+            logging.info("Merging because do_merge is %s" % self.do_merge)
+            updateDataBag(self)
+        else:
+            logging.info("Not merging because do_merge is %s" % self.do_merge)
+
     def load(self, data):
         if data is not None:
             self.data = data
             self.type = self.data["type"]
-            proc = updateDataBag(self)
+            self.update_databag()
             return
         fn = self.configCache + '/' + self.fileName
         try:
@@ -296,7 +296,7 @@ class QueueFile:
                 self.__moveFile(fn, self.configCache + "/processed")
             else:
                 os.remove(fn)
-            proc = updateDataBag(self)
+            self.update_databag()
 
     def setFile(self, name):
         self.fileName = name

systemvm/patches/debian/config/opt/cloud/bin/update_config.py

@@ -25,6 +25,7 @@ import os
 import os.path
 import configure
 import json
+from cs.CsVmPassword import *
 
 logging.basicConfig(filename='/var/log/cloud.log', level=logging.DEBUG, format='%(asctime)s  %(filename)s %(funcName)s:%(lineno)d %(message)s')
 
@@ -45,15 +46,30 @@ def finish_config():
     sys.exit(returncode)
 
 
-def process_file():
+def process(do_merge=True):
     print "[INFO] Processing JSON file %s" % sys.argv[1]
     qf = QueueFile()
     qf.setFile(sys.argv[1])
+    qf.do_merge = do_merge
     qf.load(None)
+
+    return qf
+
+
+def process_file():
+    print "[INFO] process_file"
+    qf = process()
     # Converge
     finish_config()
 
 
+def process_vmpasswd():
+    print "[INFO] process_vmpassword"
+    qf = process(False)
+    print "[INFO] Sending password to password server"
+    CsPassword(qf.getData())
+
+
 def is_guestnet_configured(guestnet_dict, keys):
 
     existing_keys = []
@@ -135,6 +151,10 @@ if sys.argv[1] == "guest_network.json":
     else:
         print "[INFO] update_config.py :: No GuestNetwork configured yet. Configuring first one now."
         process_file()
+# Bypass saving passwords and running full config/convergence, just feed passwd to passwd server and stop
+elif sys.argv[1].startswith("vm_password.json"):
+    print "[INFO] update_config.py :: Processing incoming vm_passwd file => %s" % sys.argv[1]
+    process_vmpasswd()
 else:
     print "[INFO] update_config.py :: Processing incoming file => %s" % sys.argv[1]
     process_file()

test/integration/smoke/test_vm_life_cycle.py

@@ -356,7 +356,7 @@ class TestVMLifeCycle(cloudstackTestCase):
         return
 
 
-    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false", BugId="CLOUDSTACK-6984")
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
     def test_01_stop_vm(self):
         """Test Stop Virtual Machine
         """
@@ -371,6 +371,40 @@ class TestVMLifeCycle(cloudstackTestCase):
             self.fail("Failed to stop VM: %s" % e)
         return
 
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_01_stop_vm_forced(self):
+        """Test Force Stop Virtual Machine
+        """
+        try:
+            self.small_virtual_machine.stop(self.apiclient, forced=True)
+        except Exception as e:
+            self.fail("Failed to stop VM: %s" % e)
+
+        list_vm_response = VirtualMachine.list(
+                                            self.apiclient,
+                                            id=self.small_virtual_machine.id
+                                            )
+        self.assertEqual(
+                            isinstance(list_vm_response, list),
+                            True,
+                            "Check list response returns a valid list"
+                        )
+
+        self.assertNotEqual(
+                            len(list_vm_response),
+                            0,
+                            "Check VM avaliable in List Virtual Machines"
+                        )
+
+        self.assertEqual(
+                            list_vm_response[0].state,
+                            "Stopped",
+                            "Check virtual machine is in stopped state"
+                        )
+        return
+
+
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
     def test_02_start_vm(self):
         """Test Start Virtual Machine