diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index 3985c516d30..8d1963d303a 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -987,4 +987,13 @@ public class ElasticLoadBalancerManagerImpl implements return VirtualMachineName.getSystemVmId(vmName); } + + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java index d892957a50b..6d20fb2fc31 100755 --- a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java +++ b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java @@ -345,4 +345,12 @@ public class AgentBasedConsoleProxyManager implements ConsoleProxyManager, Virtu @Override public void finalizeExpunge(ConsoleProxyVO proxy) { } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 6fef240d69d..ec4f685f3a7 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -2006,4 +2006,12 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx } return _hashKey; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index dea411484ff..766c7690ee8 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2972,4 +2972,17 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian return routerControlIpAddress; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + //asssume that if failed to ssh into router, meaning router is crashed + CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); + if (answer == null || !answer.getResult()) { + return true; + } + + return false; + } } diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 9acc702e61b..67ad97c204b 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -180,7 +180,7 @@ public interface StorageManager extends StorageService, Manager { void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated); - void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; + void prepare(VirtualMachineProfile vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; void release(VirtualMachineProfile profile); diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index baea54c663c..27d8cb71e83 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -3173,7 +3173,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } @Override - public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException { + public void prepare(VirtualMachineProfile vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException { if (dest == null) { if (s_logger.isDebugEnabled()) { @@ -3193,7 +3193,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (dest.getStorageForDisks() != null) { assignedPool = dest.getStorageForDisks().get(vol); } - if (assignedPool != null) { + if (assignedPool == null && recreate) { + assignedPool = _storagePoolDao.findById(vol.getPoolId()); + + } + if (assignedPool != null || recreate) { Volume.State state = vol.getState(); if (state == Volume.State.Allocated || state == Volume.State.Creating) { recreateVols.add(vol); @@ -3231,6 +3235,12 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag for (VolumeVO vol : recreateVols) { VolumeVO newVol; + StoragePool existingPool = null; + if (recreate && (dest.getStorageForDisks() == null || dest.getStorageForDisks().get(vol) == null)) { + existingPool = _storagePoolDao.findById(vol.getPoolId()); + s_logger.debug("existing pool: " + existingPool.getId()); + } + if (vol.getState() == Volume.State.Allocated || vol.getState() == Volume.State.Creating) { newVol = vol; } else { @@ -3251,7 +3261,9 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } catch (NoTransitionException e) { throw new CloudRuntimeException("Unable to create " + e.toString()); } - Pair created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest); + + Pair created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest, existingPool); + if (created == null) { Long poolId = newVol.getPoolId(); newVol.setPoolId(null); @@ -3307,7 +3319,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } public Pair createVolume(VolumeVO toBeCreated, DiskOfferingVO offering, VirtualMachineProfile vm, List alreadyCreated, - DeployDestination dest) throws StorageUnavailableException { + DeployDestination dest, StoragePool sPool) throws StorageUnavailableException { if (s_logger.isDebugEnabled()) { s_logger.debug("Creating volume: " + toBeCreated); } @@ -3317,9 +3329,15 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (toBeCreated.getTemplateId() != null) { template = _templateDao.findById(toBeCreated.getTemplateId()); } + + StoragePool pool = null; + if (sPool != null) { + pool = sPool; + } else { + pool = dest.getStorageForDisks().get(toBeCreated); + } - if (dest.getStorageForDisks() != null) { - StoragePool pool = dest.getStorageForDisks().get(toBeCreated); + if (pool != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to create in " + pool); } diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index bb7fa02ef22..1121bb19f61 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -1450,4 +1450,12 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V } return null; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 635a943f676..649e215acf0 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -3534,6 +3534,12 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager s_logger.info("AssignVM: vm " + vm.getInstanceName() + " now belongs to account " + cmd.getAccountName()); return vm; } + @Override + public boolean recreateNeeded(VirtualMachineProfile profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } @Override diff --git a/server/src/com/cloud/vm/VirtualMachineGuru.java b/server/src/com/cloud/vm/VirtualMachineGuru.java index d779485b3b6..b92982c7b31 100644 --- a/server/src/com/cloud/vm/VirtualMachineGuru.java +++ b/server/src/com/cloud/vm/VirtualMachineGuru.java @@ -63,6 +63,8 @@ public interface VirtualMachineGuru { void finalizeExpunge(T vm); + boolean recreateNeeded(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context); + /** * Returns the id parsed from the name. If it cannot parse the name, * then return null. This method is used to determine if this is diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 561e5b290c8..ac8fd2ab46b 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -651,6 +651,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene DataCenterDeployment originalPlan = plan; int retry = _retry; + boolean recreate = false; while (retry-- != 0) { // It's != so that it can match -1. if(reuseVolume){ @@ -745,7 +746,8 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene } _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { - _storageMgr.prepare(vmProfile, dest); + _storageMgr.prepare(vmProfile, dest, recreate); + recreate = false; } //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity if(!reuseVolume){ @@ -798,6 +800,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene if (s_logger.isDebugEnabled()) { s_logger.info("The guru did not like the answers so stopping " + vm); } + StopCommand cmd = new StopCommand(vm.getInstanceName()); StopAnswer answer = (StopAnswer) _agentMgr.easySend(destHostId, cmd); if (answer == null || !answer.getResult()) { @@ -805,7 +808,11 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene _haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop); throw new ExecutionException("Unable to stop " + vm + " so we are unable to retry the start operation"); } - throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + if (vmGuru.recreateNeeded(vmProfile, destHostId, cmds, ctx)) { + recreate = true; + } else { + throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + } } } s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails())); diff --git a/server/test/com/cloud/vm/MockUserVmManagerImpl.java b/server/test/com/cloud/vm/MockUserVmManagerImpl.java index 5241eb216a6..31065fe48a6 100644 --- a/server/test/com/cloud/vm/MockUserVmManagerImpl.java +++ b/server/test/com/cloud/vm/MockUserVmManagerImpl.java @@ -290,6 +290,12 @@ public class MockUserVmManagerImpl implements UserVmManager, UserVmService, Mana } + @Override + public boolean recreateNeeded(VirtualMachineProfile profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } @Override public UserVm startVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {