diff --git a/api/src/com/cloud/api/ApiConstants.java b/api/src/com/cloud/api/ApiConstants.java index 890188957aa..8be9e0e8d91 100755 --- a/api/src/com/cloud/api/ApiConstants.java +++ b/api/src/com/cloud/api/ApiConstants.java @@ -64,6 +64,7 @@ public class ApiConstants { public static final String ENTRY_TIME = "entrytime"; public static final String FIRSTNAME = "firstname"; public static final String FORCED = "forced"; + public static final String FORCED_DESTRORY_VM = "forcedestroyvm"; public static final String FORMAT = "format"; public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String GATEWAY = "gateway"; diff --git a/api/src/com/cloud/api/commands/DeleteHostCmd.java b/api/src/com/cloud/api/commands/DeleteHostCmd.java index 7ef817e5982..483da9de9bc 100644 --- a/api/src/com/cloud/api/commands/DeleteHostCmd.java +++ b/api/src/com/cloud/api/commands/DeleteHostCmd.java @@ -43,6 +43,9 @@ public class DeleteHostCmd extends BaseCmd { @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, description = "Force delete the host. All HA enabled vms running on the host will be put to HA; HA disabled ones will be stopped") private Boolean forced; + + @Parameter(name = ApiConstants.FORCED_DESTRORY_VM, type = CommandType.BOOLEAN, description = "Force destroy VMs on this host. All VMs created on this local storage will be destroyed") + private Boolean forceDestroyVM; // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// @@ -55,6 +58,10 @@ public class DeleteHostCmd extends BaseCmd { public boolean isForced() { return (forced != null) ? forced : false; } + + public boolean isForceDestoryVM() { + return (forceDestroyVM != null) ? forceDestroyVM : false; + } // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// @@ -72,7 +79,7 @@ public class DeleteHostCmd extends BaseCmd { @Override public void execute() { - boolean result = _resourceService.deleteHost(getId(), isForced()); + boolean result = _resourceService.deleteHost(getId(), isForced(), isForceDestoryVM()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); this.setResponseObject(response); diff --git a/api/src/com/cloud/api/commands/PreparePrimaryStorageForMaintenanceCmd.java b/api/src/com/cloud/api/commands/PreparePrimaryStorageForMaintenanceCmd.java index 354a8bb0aee..8200233fbce 100644 --- a/api/src/com/cloud/api/commands/PreparePrimaryStorageForMaintenanceCmd.java +++ b/api/src/com/cloud/api/commands/PreparePrimaryStorageForMaintenanceCmd.java @@ -99,7 +99,7 @@ public class PreparePrimaryStorageForMaintenanceCmd extends BaseAsyncCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException{ - StoragePool result = _storageService.preparePrimaryStorageForMaintenance(this); + StoragePool result = _storageService.preparePrimaryStorageForMaintenance(getId()); if (result != null){ StoragePoolResponse response = _responseGenerator.createStoragePoolResponse(result); response.setResponseName("storagepool"); diff --git a/api/src/com/cloud/resource/ResourceService.java b/api/src/com/cloud/resource/ResourceService.java index 4a3cfe9dbe0..8775bc12961 100644 --- a/api/src/com/cloud/resource/ResourceService.java +++ b/api/src/com/cloud/resource/ResourceService.java @@ -79,7 +79,7 @@ public interface ResourceService { * * @param true if deleted, false otherwise */ - boolean deleteHost(long hostId, boolean isForced); + boolean deleteHost(long hostId, boolean isForced, boolean isForceDeleteStorage); boolean updateHostPassword(UpdateHostPasswordCmd upasscmd); diff --git a/api/src/com/cloud/storage/StorageService.java b/api/src/com/cloud/storage/StorageService.java index 9bcf1139b0d..82714d5518d 100644 --- a/api/src/com/cloud/storage/StorageService.java +++ b/api/src/com/cloud/storage/StorageService.java @@ -89,7 +89,7 @@ public interface StorageService { * @throws InsufficientCapacityException * TODO */ - public StoragePool preparePrimaryStorageForMaintenance(PreparePrimaryStorageForMaintenanceCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException; + public StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException; /** * Complete maintenance for primary storage diff --git a/server/src/com/cloud/agent/AgentManager.java b/server/src/com/cloud/agent/AgentManager.java index 253ec580e11..35b4562d07b 100755 --- a/server/src/com/cloud/agent/AgentManager.java +++ b/server/src/com/cloud/agent/AgentManager.java @@ -206,7 +206,7 @@ public interface AgentManager extends Manager { * TODO * @param true if deleted, false otherwise */ - boolean deleteHost(long hostId, boolean isForced, User caller); + boolean deleteHost(long hostId, boolean isForced, boolean forceDestroy, User caller); /** * Find a pod based on the user id, template, and data center. diff --git a/server/src/com/cloud/agent/manager/AgentManagerImpl.java b/server/src/com/cloud/agent/manager/AgentManagerImpl.java index 368796595ca..836c7bf9aec 100755 --- a/server/src/com/cloud/agent/manager/AgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/AgentManagerImpl.java @@ -120,10 +120,17 @@ import com.cloud.network.dao.IPAddressDao; import com.cloud.resource.ServerResource; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StoragePoolVO; +import com.cloud.storage.StorageService; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.resource.DummySecondaryStorageResource; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.AccountManager; @@ -212,6 +219,8 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { protected ClusterDetailsDao _clusterDetailsDao = null; @Inject protected HostTagsDao _hostTagsDao = null; + @Inject + protected VolumeDao _volumeDao = null; protected int _port; @@ -225,6 +234,9 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { @Inject protected VirtualMachineManager _vmMgr = null; + + @Inject StorageService _storageSvr = null; + @Inject StorageManager _storageMgr = null; protected int _retry = 2; @@ -569,7 +581,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { @Override @DB - public boolean deleteHost(long hostId, boolean isForced, User caller) { + public boolean deleteHost(long hostId, boolean isForced, boolean forceDestroy, User caller) { // Check if the host exists HostVO host = _hostDao.findById(hostId); @@ -590,7 +602,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { // Check if host is ready for removal Status currentState = host.getStatus(); Status nextState = currentState.getNextStatus(Status.Event.Remove); - if (nextState == null && !isForced) { + if (nextState == null) { s_logger.debug("There is no transition from state " + currentState.toString() + " to state " + Status.Event.Remove.toString()); return false; } @@ -599,28 +611,54 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { s_logger.debug("Deleting Host: " + hostId + " Guid:" + host.getGuid()); } - // Check if there are vms running/starting/stopping on this host - List vms = _vmDao.listByHostId(hostId); - - if (!vms.isEmpty()) { - if (isForced) { - // Stop HA disabled vms and HA enabled vms in Stopping state - // Restart HA enabled vms - for (VMInstanceVO vm : vms) { - if (!vm.isHaEnabled() || vm.getState() == State.Stopping) { - s_logger.debug("Stopping vm: " + vm + " as a part of deleteHost id=" + hostId); - if (!_vmMgr.advanceStop(vm, true, caller, _accountMgr.getAccount(vm.getAccountId()))) { - String errorMsg = "There was an error stopping the vm: " + vm + " as a part of hostDelete id=" + hostId; - s_logger.warn(errorMsg); - throw new CloudRuntimeException(errorMsg); + if (forceDestroy) { + //put local storage into mainenance mode, will set all the VMs on this local storage into stopped state + StoragePool storagePool = _storageMgr.findLocalStorageOnHost(host.getId()); + if (storagePool != null) { + if (storagePool.getStatus() == StoragePoolStatus.Up || storagePool.getStatus() == StoragePoolStatus.ErrorInMaintenance) { + try { + storagePool = _storageSvr.preparePrimaryStorageForMaintenance(storagePool.getId()); + if (storagePool == null) { + s_logger.debug("Failed to set primary storage into maintenance mode"); + return false; } - } else if (vm.isHaEnabled() && (vm.getState() == State.Running || vm.getState() == State.Starting)) { - s_logger.debug("Scheduling restart for vm: " + vm + " " + vm.getState() + " on the host id=" + hostId); - _haMgr.scheduleRestart(vm, false); + } catch (Exception e) { + s_logger.debug("Failed to set primary storage into maintenance mode, due to: " + e.toString()); + return false; } } - } else { - throw new CloudRuntimeException("Unable to delete the host as there are vms in " + vms.get(0).getState() + " state using this host and isForced=false specified"); + List vmsOnLocalStorage = _storageMgr.listByStoragePool(storagePool.getId()); + for (VMInstanceVO vm : vmsOnLocalStorage) { + if(!_vmMgr.destroy(vm, caller, _accountMgr.getAccount(vm.getAccountId()))) { + String errorMsg = "There was an error Destory the vm: " + vm + " as a part of hostDelete id=" + hostId; + s_logger.warn(errorMsg); + throw new CloudRuntimeException(errorMsg); + } + } + } + } else { + // Check if there are vms running/starting/stopping on this host + List vms = _vmDao.listByHostId(hostId); + if (!vms.isEmpty()) { + if (isForced) { + // Stop HA disabled vms and HA enabled vms in Stopping state + // Restart HA enabled vms + for (VMInstanceVO vm : vms) { + if (!vm.isHaEnabled() || vm.getState() == State.Stopping) { + s_logger.debug("Stopping vm: " + vm + " as a part of deleteHost id=" + hostId); + if (!_vmMgr.advanceStop(vm, true, caller, _accountMgr.getAccount(vm.getAccountId()))) { + String errorMsg = "There was an error stopping the vm: " + vm + " as a part of hostDelete id=" + hostId; + s_logger.warn(errorMsg); + throw new CloudRuntimeException(errorMsg); + } + } else if (vm.isHaEnabled() && (vm.getState() == State.Running || vm.getState() == State.Starting)) { + s_logger.debug("Scheduling restart for vm: " + vm + " " + vm.getState() + " on the host id=" + hostId); + _haMgr.scheduleRestart(vm, false); + } + } + } else { + throw new CloudRuntimeException("Unable to delete the host as there are vms in " + vms.get(0).getState() + " state using this host and isForced=false specified"); + } } } @@ -701,7 +739,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { for (StoragePoolHostVO pool : pools) { Long poolId = pool.getPoolId(); StoragePoolVO storagePool = _storagePoolDao.findById(poolId); - if (storagePool.isLocal()) { + if (storagePool.isLocal() && forceDestroy) { storagePool.setUuid(null); storagePool.setClusterId(null); _storagePoolDao.update(poolId, storagePool); @@ -1510,7 +1548,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager { return cancelMaintenance(hostId); } else if (event == Event.Remove) { User caller = _accountMgr.getActiveUser(User.UID_SYSTEM); - return deleteHost(hostId, false, caller); + return deleteHost(hostId, false, false, caller); } else if (event == Event.AgentDisconnected) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received agent disconnect event for host " + hostId); diff --git a/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 6e3338853de..eeb2dfa932f 100644 --- a/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -356,7 +356,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust @Override @DB - public boolean deleteHost(long hostId, boolean isForced, User caller) { + public boolean deleteHost(long hostId, boolean isForced, boolean forceDestroy, User caller) { try { Boolean result = _clusterMgr.propagateAgentEvent(hostId, Event.Remove); if (result != null) { @@ -366,7 +366,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return false; } - return super.deleteHost(hostId, isForced, caller); + return super.deleteHost(hostId, isForced, forceDestroy, caller); } @Override diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index fcc9e1aa0d9..0620341786b 100644 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -529,7 +529,7 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma } @Override - public boolean deleteHost(long hostId, boolean isForced) { + public boolean deleteHost(long hostId, boolean isForced, boolean forceDestroy) { User caller = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); // Verify that host exists HostVO host = _hostDao.findById(hostId); @@ -541,7 +541,7 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma _secondaryStorageMgr.deleteHost(hostId); return true; } else { - return _agentMgr.deleteHost(hostId, isForced, caller); + return _agentMgr.deleteHost(hostId, isForced, forceDestroy, caller); } } diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index a3e4b31f87c..e4e14ca02c6 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -202,4 +202,7 @@ public interface StorageManager extends Manager { List ListByDataCenterHypervisor(long datacenterId, HypervisorType type); VMTemplateHostVO getTemplateHostRef(long zoneId, long tmpltId, boolean readyOnly); + List listByStoragePool(long storagePoolId); + + StoragePoolVO findLocalStorageOnHost(long hostId); } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index c502c80b0ec..e476c68fb3b 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -282,6 +282,8 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag protected StoragePoolWorkDao _storagePoolWorkDao; @Inject protected HypervisorGuruManager _hvGuruMgr; + @Inject + protected VolumeDao _volumeDao; @Inject(adapter = StoragePoolAllocator.class) protected Adapters _storagePoolAllocators; @@ -290,6 +292,8 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag protected SearchBuilder HostTemplateStatesSearch; protected GenericSearchBuilder UpHostsInPoolSearch; + protected SearchBuilder StoragePoolSearch; + protected SearchBuilder LocalStorageSearch; ScheduledExecutorService _executor = null; boolean _storageCleanupEnabled; @@ -864,6 +868,20 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag UpHostsInPoolSearch.and("pool", UpHostsInPoolSearch.entity().getPoolId(), Op.EQ); UpHostsInPoolSearch.done(); + StoragePoolSearch = _vmInstanceDao.createSearchBuilder(); + + SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); + volumeSearch.and("volumeType", volumeSearch.entity().getVolumeType(), SearchCriteria.Op.EQ); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ); + StoragePoolSearch.join("vmVolume", volumeSearch, volumeSearch.entity().getInstanceId(), StoragePoolSearch.entity().getId(), JoinBuilder.JoinType.INNER); + StoragePoolSearch.done(); + + LocalStorageSearch = _storagePoolDao.createSearchBuilder(); + SearchBuilder storageHostSearch = _storagePoolHostDao.createSearchBuilder(); + storageHostSearch.and("hostId", storageHostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + LocalStorageSearch.join("poolHost", storageHostSearch, storageHostSearch.entity().getPoolId(), LocalStorageSearch.entity().getId(), JoinBuilder.JoinType.INNER); + LocalStorageSearch.and("type", LocalStorageSearch.entity().getPoolType(), SearchCriteria.Op.IN); + LocalStorageSearch.done(); return true; } @@ -2019,8 +2037,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag @Override @DB - public StoragePoolVO preparePrimaryStorageForMaintenance(PreparePrimaryStorageForMaintenanceCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException { - Long primaryStorageId = cmd.getId(); + public StoragePoolVO preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException { Long userId = UserContext.current().getCallerUserId(); User user = _userDao.findById(userId); Account account = UserContext.current().getCaller(); @@ -2090,6 +2107,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } } + // 4. Process the queue List pendingWork = _storagePoolWorkDao.listPendingWorkForPrepareForMaintenanceByPoolId(primaryStorageId); @@ -2985,8 +3003,10 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag long dcId = pool.getDataCenterId(); Long podId = pool.getPodId(); + List secHosts = _hostDao.listSecondaryStorageHosts(dcId); + //FIXME, for cloudzone, the local secondary storoge - if (pool.isLocal() && pool.getPoolType() == StoragePoolType.Filesystem) { + if (pool.isLocal() && pool.getPoolType() == StoragePoolType.Filesystem && secHosts.isEmpty()) { List sphs = _storagePoolHostDao.listByPoolId(pool.getId()); if (!sphs.isEmpty()) { StoragePoolHostVO localStoragePoolHost = sphs.get(0); @@ -2996,7 +3016,6 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } } - List secHosts = _hostDao.listSecondaryStorageHosts(dcId); if (secHosts.size() == 1) { VMTemplateHostVO templateHostVO = _templateHostDao.findByHostTemplate(secHosts.get(0).getId(), templateId); return templateHostVO; @@ -3015,5 +3034,28 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } return null; } + + + @Override + @DB + public List listByStoragePool(long storagePoolId) { + SearchCriteria sc = StoragePoolSearch.create(); + sc.setJoinParameters("vmVolume", "volumeType", Volume.Type.ROOT); + sc.setJoinParameters("vmVolume", "poolId", storagePoolId); + return _vmInstanceDao.search(sc, null); + } + @Override + @DB + public StoragePoolVO findLocalStorageOnHost(long hostId) { + SearchCriteria sc = LocalStorageSearch.create(); + sc.setParameters("type", new Object[]{StoragePoolType.Filesystem, StoragePoolType.LVM}); + sc.setJoinParameters("poolHost", "hostId", hostId); + List storagePools = _storagePoolDao.search(sc, null); + if (!storagePools.isEmpty()) { + return storagePools.get(0); + } else { + return null; + } + } } diff --git a/server/src/com/cloud/storage/dao/StoragePoolDao.java b/server/src/com/cloud/storage/dao/StoragePoolDao.java index f33e26a6e0d..2d943e55f17 100644 --- a/server/src/com/cloud/storage/dao/StoragePoolDao.java +++ b/server/src/com/cloud/storage/dao/StoragePoolDao.java @@ -105,5 +105,6 @@ public interface StoragePoolDao extends GenericDao { List findIfDuplicatePoolsExistByUUID(String uuid); - List listPoolsByStatus(StoragePoolStatus status); + List listPoolsByStatus(StoragePoolStatus status); + }