CLOUDSTACK-7742:

root cause:
when vmsync reports system VM is down, CCP doesn't release the VM resource before starting it.
fix:
make sure cleanup is called for a VM when it is reported as Stopped

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1283,6 +1283,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         s_logger.debug("Cleaning up resources for the vm " + vm + " in " + state + " state");
         try {
             if (state == State.Starting) {
+                if (work != null) {
                     Step step = work.getStep();
                     if (step == Step.Starting && !cleanUpEvenIfUnableToStop) {
                         s_logger.warn("Unable to cleanup vm " + vm + "; work state is incorrect: " + step);
@@ -1302,6 +1303,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                         s_logger.debug("Cleanup is not needed for vm " + vm + "; work state is incorrect: " + step);
                         return true;
                      }
+                } else {
+                    if (vm.getHostId() != null) {
+                        if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) {
+                            s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process");
+                            return false;
+                        }
+                    }
+                }
+
             } else if (state == State.Stopping) {
                 if (vm.getHostId() != null) {
                     if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) {
@@ -1453,20 +1463,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
             if (!cleanUpEvenIfUnableToStop) {
                 throw new CloudRuntimeException("We cannot stop " + vm + " when it is in state " + vm.getState());
             }
-            boolean doCleanup = false;
+            boolean doCleanup = true;
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("Unable to transition the state but we're moving on because it's forced stop");
             }
-            if ((state == State.Starting) || (state == State.Migrating) || (state == State.Stopping)) {
-                if (work != null) {
-                    doCleanup = true;
-                } else {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Unable to cleanup VM: " + vm + " ,since outstanding work item is not found");
-                    }
-                    throw new CloudRuntimeException("Work item not found, We cannot stop " + vm + " when it is in state " + vm.getState());
-                }
-            }
 
             if (doCleanup) {
                 if (cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, cleanUpEvenIfUnableToStop)) {

engine/schema/src/com/cloud/dc/dao/DataCenterDaoImpl.java

@@ -250,6 +250,7 @@ public class DataCenterDaoImpl extends GenericDaoBase<DataCenterVO, Long> implem
 
     @Override
     public Pair<String, Long> allocatePrivateIpAddress(long dcId, long podId, long instanceId, String reservationId) {
+        _ipAllocDao.releaseIpAddress(instanceId);
         DataCenterIpAddressVO vo = _ipAllocDao.takeIpAddress(dcId, podId, instanceId, reservationId);
         if (vo == null) {
             return null;

engine/schema/src/com/cloud/dc/dao/DataCenterIpAddressDao.java

@@ -33,6 +33,8 @@ public interface DataCenterIpAddressDao extends GenericDao<DataCenterIpAddressVO
 
     public void releaseIpAddress(long nicId, String reservationId);
 
+    public void releaseIpAddress(long nicId);
+
     boolean mark(long dcId, long podId, String ip);
 
     List<DataCenterIpAddressVO> listByPodIdDcIdIpAddress(long podId, long dcId, String ipAddress);

engine/schema/src/com/cloud/dc/dao/DataCenterIpAddressDaoImpl.java

@@ -177,6 +177,22 @@ public class DataCenterIpAddressDaoImpl extends GenericDaoBase<DataCenterIpAddre
         update(vo, sc);
     }
 
+    @Override
+    public void releaseIpAddress(long nicId) {
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Releasing ip address for instance=" + nicId);
+        }
+        SearchCriteria<DataCenterIpAddressVO> sc = AllFieldsSearch.create();
+        sc.setParameters("instance", nicId);
+
+        DataCenterIpAddressVO vo = createForUpdate();
+        vo.setTakenAt(null);
+        vo.setInstanceId(null);
+        vo.setReservationId(null);
+        update(vo, sc);
+    }
+
+
     @Override
     public List<DataCenterIpAddressVO> listByPodIdDcId(long podId, long dcId) {
         SearchCriteria<DataCenterIpAddressVO> sc = AllFieldsSearch.create();