CS-15544: recreate router if router filesystem is crashed

plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java

@@ -987,4 +987,13 @@ public class ElasticLoadBalancerManagerImpl implements
 
         return VirtualMachineName.getSystemVmId(vmName);
     }
+
+
+	@Override
+	public boolean recreateNeeded(
+			VirtualMachineProfile<DomainRouterVO> profile, long hostId,
+			Commands cmds, ReservationContext context) {
+		// TODO Auto-generated method stub
+		return false;
+	}
 }

server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java

@@ -345,4 +345,12 @@ public class AgentBasedConsoleProxyManager implements ConsoleProxyManager, Virtu
     @Override 
     public void finalizeExpunge(ConsoleProxyVO proxy) {
     }
+
+	@Override
+	public boolean recreateNeeded(
+			VirtualMachineProfile<ConsoleProxyVO> profile, long hostId,
+			Commands cmds, ReservationContext context) {
+		// TODO Auto-generated method stub
+		return false;
+	}
 }

server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -2006,4 +2006,12 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx
         }
         return _hashKey;
     }
+
+	@Override
+	public boolean recreateNeeded(
+			VirtualMachineProfile<ConsoleProxyVO> profile, long hostId,
+			Commands cmds, ReservationContext context) {
+		// TODO Auto-generated method stub
+		return false;
+	}
 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -2972,4 +2972,17 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
             
         return routerControlIpAddress;
     }
+
+	@Override
+	public boolean recreateNeeded(
+			VirtualMachineProfile<DomainRouterVO> profile, long hostId,
+			Commands cmds, ReservationContext context) {
+		//asssume that if failed to ssh into router, meaning router is crashed
+		CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh");
+		if (answer == null || !answer.getResult()) {
+			return true;
+		}
+
+		return false;
+	}
 }

server/src/com/cloud/storage/StorageManager.java

@@ -180,7 +180,7 @@ public interface StorageManager extends StorageService, Manager {
 	void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated);
 
     
-    void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException;
+    void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException;
 
 	void release(VirtualMachineProfile<? extends VMInstanceVO> profile);
 

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -3173,7 +3173,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
     }
 
     @Override
-    public void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException {
+    public void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException {
 
         if (dest == null) {
             if (s_logger.isDebugEnabled()) {
@@ -3193,7 +3193,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
             if (dest.getStorageForDisks() != null) {
                 assignedPool = dest.getStorageForDisks().get(vol);
             }
-            if (assignedPool != null) {
+            if (assignedPool == null && recreate) {
+            	assignedPool = _storagePoolDao.findById(vol.getPoolId());
+            	
+            }
+            if (assignedPool != null || recreate) {
                 Volume.State state = vol.getState();
                 if (state == Volume.State.Allocated || state == Volume.State.Creating) {
                     recreateVols.add(vol);
@@ -3231,6 +3235,12 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
 
         for (VolumeVO vol : recreateVols) {
             VolumeVO newVol;
+            StoragePool existingPool = null;
+            if (recreate && (dest.getStorageForDisks() == null || dest.getStorageForDisks().get(vol) == null)) {
+            	existingPool = _storagePoolDao.findById(vol.getPoolId());
+            	s_logger.debug("existing pool: " + existingPool.getId());
+            }
+            
             if (vol.getState() == Volume.State.Allocated || vol.getState() == Volume.State.Creating) {
                 newVol = vol;
             } else {
@@ -3251,7 +3261,9 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
             } catch (NoTransitionException e) {
                 throw new CloudRuntimeException("Unable to create " + e.toString());
             }
-            Pair<VolumeTO, StoragePool> created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest);
+
+            Pair<VolumeTO, StoragePool> created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest, existingPool);
+
             if (created == null) {
                 Long poolId = newVol.getPoolId();
                 newVol.setPoolId(null);
@@ -3307,7 +3319,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
     }
 
     public Pair<VolumeTO, StoragePool> createVolume(VolumeVO toBeCreated, DiskOfferingVO offering, VirtualMachineProfile<? extends VirtualMachine> vm, List<? extends Volume> alreadyCreated,
-            DeployDestination dest) throws StorageUnavailableException {
+            DeployDestination dest, StoragePool sPool) throws StorageUnavailableException {
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Creating volume: " + toBeCreated);
         }
@@ -3318,8 +3330,14 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
             template = _templateDao.findById(toBeCreated.getTemplateId());
         }
         
-        if (dest.getStorageForDisks() != null) {
+        StoragePool pool = null;
-            StoragePool pool = dest.getStorageForDisks().get(toBeCreated);
+        if (sPool != null) {
+        	pool = sPool;
+        } else {
+        	pool = dest.getStorageForDisks().get(toBeCreated);
+        }
+
+        if (pool != null) {
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("Trying to create in " + pool);
             }

server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java

@@ -1450,4 +1450,12 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
         }
         return null;
     }
+
+	@Override
+	public boolean recreateNeeded(
+			VirtualMachineProfile<SecondaryStorageVmVO> profile, long hostId,
+			Commands cmds, ReservationContext context) {
+		// TODO Auto-generated method stub
+		return false;
+	}
 }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -3534,6 +3534,12 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         s_logger.info("AssignVM: vm " + vm.getInstanceName() + " now belongs to account " + cmd.getAccountName());
         return vm;
     }
+    @Override
+    public boolean recreateNeeded(VirtualMachineProfile<UserVmVO> profile,
+                       long hostId, Commands cmds, ReservationContext context) {
+               // TODO Auto-generated method stub
+               return false;
+    }
 
 
     @Override

server/src/com/cloud/vm/VirtualMachineGuru.java

@@ -63,6 +63,8 @@ public interface VirtualMachineGuru<T extends VirtualMachine> {
     
     void finalizeExpunge(T vm);
     
+    boolean recreateNeeded(VirtualMachineProfile<T> profile, long hostId, Commands cmds, ReservationContext context);
+    
     /**
      * Returns the id parsed from the name.  If it cannot parse the name,
      * then return null.  This method is used to determine if this is

server/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -651,6 +651,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
             DataCenterDeployment originalPlan = plan;
 
             int retry = _retry;
+            boolean recreate = false;
             while (retry-- != 0) { // It's != so that it can match -1.
 
                 if(reuseVolume){
@@ -745,7 +746,8 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
                     }
                     _networkMgr.prepare(vmProfile, dest, ctx); 
                     if (vm.getHypervisorType() != HypervisorType.BareMetal) {
-                        _storageMgr.prepare(vmProfile, dest);
+                        _storageMgr.prepare(vmProfile, dest, recreate);
+                        recreate = false;
                     }
                     //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity
                     if(!reuseVolume){
@@ -798,6 +800,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
                             if (s_logger.isDebugEnabled()) {
                                 s_logger.info("The guru did not like the answers so stopping " + vm);
                             }
+                           
                             StopCommand cmd = new StopCommand(vm.getInstanceName());
                             StopAnswer answer = (StopAnswer) _agentMgr.easySend(destHostId, cmd);
                             if (answer == null || !answer.getResult()) {
@@ -805,7 +808,11 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
                                 _haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop);
                                 throw new ExecutionException("Unable to stop " + vm + " so we are unable to retry the start operation");
                             }
-                            throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying");
+                            if (vmGuru.recreateNeeded(vmProfile, destHostId, cmds, ctx)) {
+                            	recreate = true;
+                            } else {
+                            	throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying");
+                            }
                         }
                     }
                     s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails()));

server/test/com/cloud/vm/MockUserVmManagerImpl.java

@@ -290,6 +290,12 @@ public class MockUserVmManagerImpl implements UserVmManager, UserVmService, Mana
     }
 
 
+        @Override
+       public boolean recreateNeeded(VirtualMachineProfile<UserVmVO> profile,
+                       long hostId, Commands cmds, ReservationContext context) {
+               // TODO Auto-generated method stub
+               return false;
+       }
 
     @Override
     public UserVm startVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {