refactor storapool automation (#11789)

Co-authored-by: Daan Hoogland <dahn@apache.org>

server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java

@@ -18,19 +18,18 @@
  */
 package com.cloud.storage;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.context.CallContext;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 import org.springframework.stereotype.Component;
@@ -39,32 +38,18 @@ import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.ModifyStoragePoolCommand;
 import com.cloud.agent.api.ModifyStoragePoolAnswer;
-import com.cloud.alert.AlertManager;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceManager;
 import com.cloud.server.ManagementServer;
-import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.StoragePoolWorkDao;
 import com.cloud.storage.dao.VolumeDao;
-import com.cloud.user.Account;
-import com.cloud.user.User;
-import com.cloud.user.dao.UserDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.vm.ConsoleProxyVO;
-import com.cloud.vm.DomainRouterVO;
-import com.cloud.vm.SecondaryStorageVmVO;
-import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
-import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineManager;
-import com.cloud.vm.dao.ConsoleProxyDao;
-import com.cloud.vm.dao.DomainRouterDao;
-import com.cloud.vm.dao.SecondaryStorageVmDao;
-import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
 
 @Component
@@ -72,30 +57,11 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
     protected Logger logger = LogManager.getLogger(getClass());
     @Inject
     protected VirtualMachineManager vmMgr;
-    @Inject
-    protected SecondaryStorageVmDao _secStrgDao;
-    @Inject
-    UserVmDao userVmDao;
-    @Inject
-    protected UserDao _userDao;
-    @Inject
-    protected DomainRouterDao _domrDao;
-    @Inject
-    protected StoragePoolHostDao _storagePoolHostDao;
-    @Inject
-    protected AlertManager _alertMgr;
-    @Inject
-    protected ConsoleProxyDao _consoleProxyDao;
-
     @Inject
     protected StoragePoolWorkDao _storagePoolWorkDao;
     @Inject
     PrimaryDataStoreDao primaryDataStoreDao;
     @Inject
-    StoragePoolDetailsDao storagePoolDetailsDao;
-    @Inject
-    DataStoreManager dataStoreMgr;
-    @Inject
     protected ResourceManager _resourceMgr;
     @Inject
     AgentManager agentMgr;
@@ -106,192 +72,36 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
     @Inject
     ManagementServer server;
     @Inject
-    DataStoreProviderManager providerMgr;
-    @Inject
     StorageManager storageManager;
 
     @Override
     public boolean maintain(DataStore store) {
-        Long userId = CallContext.current().getCallingUserId();
-        User user = _userDao.findById(userId);
-        Account account = CallContext.current().getCallingAccount();
         StoragePoolVO pool = primaryDataStoreDao.findById(store.getId());
         try {
-            List<StoragePoolVO> spes = null;
+            getStoragePoolForSpecification(pool);
-            // Handling Zone and Cluster wide storage scopes.
-            // if the storage is ZONE wide then we pass podid and cluster id as null as they will be empty for ZWPS
-            if (pool.getScope() == ScopeType.ZONE) {
-                spes = primaryDataStoreDao.listBy(pool.getDataCenterId(), null, null, ScopeType.ZONE);
-            } else {
-                spes = primaryDataStoreDao.listBy(pool.getDataCenterId(), pool.getPodId(), pool.getClusterId(), ScopeType.CLUSTER);
-            }
-            for (StoragePoolVO sp : spes) {
-                if (sp.getParent() != pool.getParent() && sp.getId() != pool.getParent()) { // If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode
-                    if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) {
-                        throw new CloudRuntimeException(String.format("Only one storage pool in a cluster can be in PrepareForMaintenance mode, %s is already in  PrepareForMaintenance mode ", sp));
-                    }
-                }
-            }
-            StoragePool storagePool = (StoragePool)store;
 
-            //Handeling the Zone wide and cluster wide primay storage
+            List<HostVO> hosts = getHostsForStoragePool(pool);
-            List<HostVO> hosts = new ArrayList<HostVO>();
-            // if the storage scope is ZONE wide, then get all the hosts for which hypervisor ZWSP created to send Modifystoragepoolcommand
-            //TODO: if it's zone wide, this code will list a lot of hosts in the zone, which may cause performance/OOM issue.
-            if (pool.getScope().equals(ScopeType.ZONE)) {
-                if (HypervisorType.Any.equals(pool.getHypervisor())) {
-                    hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZone(pool.getDataCenterId());
-                }
-                else {
-                    hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(pool.getHypervisor(), pool.getDataCenterId());
-                }
-            } else {
-                hosts = _resourceMgr.listHostsInClusterByStatus(pool.getClusterId(), Status.Up);
-            }
 
-            if (hosts == null || hosts.size() == 0) {
+            if (setNextStateForMaintenance(hosts, pool) == StoragePoolStatus.PrepareForMaintenance) {
-                pool.setStatus(StoragePoolStatus.Maintenance);
+                removeHeartbeatForHostsFromPool(hosts, pool);
-                primaryDataStoreDao.update(pool.getId(), pool);
+                // check to see if other ps exist
-                return true;
+                // if they do, then we can migrate over the system vms to them
-            } else {
+                // if they don't, then just stop all vms on this one
-                // set the pool state to prepare for maintenance
+                List<StoragePoolVO> upPools = primaryDataStoreDao.listByStatusInZone(pool.getDataCenterId(), StoragePoolStatus.Up);
-                pool.setStatus(StoragePoolStatus.PrepareForMaintenance);
+                boolean restart = !CollectionUtils.isEmpty(upPools);
-                primaryDataStoreDao.update(pool.getId(), pool);
-            }
-            // remove heartbeat
-            for (HostVO host : hosts) {
-                ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(false, storagePool);
-                final Answer answer = agentMgr.easySend(host.getId(), cmd);
-                if (answer == null || !answer.getResult()) {
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("ModifyStoragePool false failed due to " + ((answer == null) ? "answer null" : answer.getDetails()));
-                    }
-                } else {
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("ModifyStoragePool false succeeded");
-                    }
-                    if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) {
-                        logger.debug("Started synchronising datastore cluster storage pool {} with vCenter", pool);
-                        storageManager.syncDatastoreClusterStoragePool(pool.getId(), ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), host.getId());
-                    }
-                }
-            }
-            // check to see if other ps exist
-            // if they do, then we can migrate over the system vms to them
-            // if they don't, then just stop all vms on this one
-            List<StoragePoolVO> upPools = primaryDataStoreDao.listByStatusInZone(pool.getDataCenterId(), StoragePoolStatus.Up);
-            boolean restart = true;
-            if (upPools == null || upPools.size() == 0) {
-                restart = false;
-            }
 
-            // 2. Get a list of all the ROOT volumes within this storage pool
+                // 2. Get a list of all the ROOT volumes within this storage pool
-            List<VolumeVO> allVolumes = volumeDao.findByPoolId(pool.getId());
+                List<VolumeVO> allVolumes = volumeDao.findByPoolId(pool.getId());
-
+                // 3. Enqueue to the work queue
-            // 3. Enqueue to the work queue
+                enqueueMigrationsForVolumes(allVolumes, pool);
-            for (VolumeVO volume : allVolumes) {
+                // 4. Process the queue
-                VMInstanceVO vmInstance = vmDao.findById(volume.getInstanceId());
+                processMigrationWorkloads(pool, restart);
-
-                if (vmInstance == null) {
-                    continue;
-                }
-
-                // enqueue sp work
-                if (vmInstance.getState().equals(State.Running) || vmInstance.getState().equals(State.Starting) || vmInstance.getState().equals(State.Stopping)) {
-
-                    try {
-                        StoragePoolWorkVO work = new StoragePoolWorkVO(vmInstance.getId(), pool.getId(), false, false, server.getId());
-                        _storagePoolWorkDao.persist(work);
-                    } catch (Exception e) {
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("Work record already exists, re-using by re-setting values");
-                        }
-                        StoragePoolWorkVO work = _storagePoolWorkDao.findByPoolIdAndVmId(pool.getId(), vmInstance.getId());
-                        work.setStartedAfterMaintenance(false);
-                        work.setStoppedForMaintenance(false);
-                        work.setManagementServerId(server.getId());
-                        _storagePoolWorkDao.update(work.getId(), work);
-                    }
-                }
-            }
-
-            // 4. Process the queue
-            List<StoragePoolWorkVO> pendingWork = _storagePoolWorkDao.listPendingWorkForPrepareForMaintenanceByPoolId(pool.getId());
-
-            for (StoragePoolWorkVO work : pendingWork) {
-                // shut down the running vms
-                VMInstanceVO vmInstance = vmDao.findById(work.getVmId());
-
-                if (vmInstance == null) {
-                    continue;
-                }
-
-                // if the instance is of type consoleproxy, call the console
-                // proxy
-                if (vmInstance.getType().equals(VirtualMachine.Type.ConsoleProxy)) {
-                    // call the consoleproxymanager
-                    ConsoleProxyVO consoleProxy = _consoleProxyDao.findById(vmInstance.getId());
-                    vmMgr.advanceStop(consoleProxy.getUuid(), false);
-                    // update work status
-                    work.setStoppedForMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-
-                    if (restart) {
-
-                        vmMgr.advanceStart(consoleProxy.getUuid(), null, null);
-                        // update work status
-                        work.setStartedAfterMaintenance(true);
-                        _storagePoolWorkDao.update(work.getId(), work);
-                    }
-                }
-
-                // if the instance is of type uservm, call the user vm manager
-                if (vmInstance.getType() == VirtualMachine.Type.User) {
-                    UserVmVO userVm = userVmDao.findById(vmInstance.getId());
-                    vmMgr.advanceStop(userVm.getUuid(), false);
-                    // update work status
-                    work.setStoppedForMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-                }
-
-                // if the instance is of type secondary storage vm, call the
-                // secondary storage vm manager
-                if (vmInstance.getType().equals(VirtualMachine.Type.SecondaryStorageVm)) {
-                    SecondaryStorageVmVO secStrgVm = _secStrgDao.findById(vmInstance.getId());
-                    vmMgr.advanceStop(secStrgVm.getUuid(), false);
-                    // update work status
-                    work.setStoppedForMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-
-                    if (restart) {
-                        vmMgr.advanceStart(secStrgVm.getUuid(), null, null);
-                        // update work status
-                        work.setStartedAfterMaintenance(true);
-                        _storagePoolWorkDao.update(work.getId(), work);
-                    }
-                }
-
-                // if the instance is of type domain router vm, call the network
-                // manager
-                if (vmInstance.getType().equals(VirtualMachine.Type.DomainRouter)) {
-                    DomainRouterVO domR = _domrDao.findById(vmInstance.getId());
-                    vmMgr.advanceStop(domR.getUuid(), false);
-                    // update work status
-                    work.setStoppedForMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-
-                    if (restart) {
-                        vmMgr.advanceStart(domR.getUuid(), null, null);
-                        // update work status
-                        work.setStartedAfterMaintenance(true);
-                        _storagePoolWorkDao.update(work.getId(), work);
-                    }
-                }
             }
         } catch (Exception e) {
             logger.error("Exception in enabling primary storage maintenance:", e);
             pool.setStatus(StoragePoolStatus.ErrorInMaintenance);
             primaryDataStoreDao.update(pool.getId(), pool);
+            // TODO decide on what recovery is possible
             throw new CloudRuntimeException(e.getMessage());
         }
         return true;
@@ -300,64 +110,64 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
     @Override
     public boolean cancelMaintain(DataStore store) {
         // Change the storage state back to up
-        Long userId = CallContext.current().getCallingUserId();
-        User user = _userDao.findById(userId);
-        Account account = CallContext.current().getCallingAccount();
         StoragePoolVO poolVO = primaryDataStoreDao.findById(store.getId());
         StoragePool pool = (StoragePool)store;
 
-        //Handeling the Zone wide and cluster wide primay storage
+        List<HostVO> hosts = getHostsForStoragePool(poolVO);
-        List<HostVO> hosts = new ArrayList<HostVO>();
-        // if the storage scope is ZONE wide, then get all the hosts for which hypervisor ZWSP created to send Modifystoragepoolcommand
-        if (poolVO.getScope().equals(ScopeType.ZONE)) {
-            if (HypervisorType.Any.equals(pool.getHypervisor())) {
-                hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZone(pool.getDataCenterId());
-            }
-            else {
-                hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(poolVO.getHypervisor(), pool.getDataCenterId());
-            }
-        } else {
-            hosts = _resourceMgr.listHostsInClusterByStatus(pool.getClusterId(), Status.Up);
-        }
 
-        if (hosts == null || hosts.size() == 0) {
+        if (CollectionUtils.isEmpty(hosts)) {
             return true;
         }
 
         Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
-        // add heartbeat
+        addHeartbeatToHostsInPool(hosts, pool, nfsMountOpts);
-        for (HostVO host : hosts) {
-            ModifyStoragePoolCommand msPoolCmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
-            final Answer answer = agentMgr.easySend(host.getId(), msPoolCmd);
-            if (answer == null || !answer.getResult()) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("ModifyStoragePool add failed due to " + ((answer == null) ? "answer null" : answer.getDetails()));
-                }
-                if (answer != null && nfsMountOpts.second()) {
-                    logger.error(String.format("Unable to attach storage pool to the host %s due to %s",  host,  answer.getDetails()));
-                    StringBuilder exceptionSB = new StringBuilder("Unable to attach storage pool to the host ").append(host.getName());
-                    String reason = storageManager.getStoragePoolMountFailureReason(answer.getDetails());
-                    if (reason!= null) {
-                        exceptionSB.append(". ").append(reason).append(".");
-                    }
-                    throw new CloudRuntimeException(exceptionSB.toString());
-                }
-            } else {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("ModifyStoragePool add succeeded");
-                }
-                storageManager.updateStoragePoolHostVOAndBytes(pool, host.getId(), (ModifyStoragePoolAnswer) answer);
-                if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) {
-                    logger.debug("Started synchronising datastore cluster storage pool {} with vCenter", pool);
-                    storageManager.syncDatastoreClusterStoragePool(pool.getId(), ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), host.getId());
-                }
-            }
-        }
 
         // 2. Get a list of pending work for this queue
         List<StoragePoolWorkVO> pendingWork = _storagePoolWorkDao.listPendingWorkForCancelMaintenanceByPoolId(poolVO.getId());
 
         // 3. work through the queue
+        cancelMigrationWorkloads(pendingWork);
+        return false;
+    }
+
+    private StoragePoolStatus setNextStateForMaintenance(List<HostVO> hosts, StoragePoolVO pool) {
+        if (CollectionUtils.isEmpty(hosts)) {
+            pool.setStatus(StoragePoolStatus.Maintenance);
+            primaryDataStoreDao.update(pool.getId(), pool);
+            return StoragePoolStatus.Maintenance;
+        } else {
+            // set the pool state to prepare for maintenance
+            pool.setStatus(StoragePoolStatus.PrepareForMaintenance);
+            primaryDataStoreDao.update(pool.getId(), pool);
+            return StoragePoolStatus.PrepareForMaintenance;
+        }
+    }
+
+    private void processMigrationWorkloads(StoragePoolVO pool, boolean restart) throws ResourceUnavailableException, OperationTimedoutException, InsufficientCapacityException {
+        List<StoragePoolWorkVO> pendingWork = _storagePoolWorkDao.listPendingWorkForPrepareForMaintenanceByPoolId(pool.getId());
+
+        for (StoragePoolWorkVO work : pendingWork) {
+            // shut down the running vms
+            VMInstanceVO vmInstance = vmDao.findById(work.getVmId());
+
+            if (vmInstance == null) {
+                continue;
+            }
+
+            switch (vmInstance.getType()) {
+                case ConsoleProxy:
+                case SecondaryStorageVm:
+                case DomainRouter:
+                    handleVmMigration(restart, work, vmInstance);
+                    break;
+                case User:
+                    handleStopVmForMigration(work, vmInstance);
+                    break;
+            }
+        }
+    }
+
+    private void cancelMigrationWorkloads(List<StoragePoolWorkVO> pendingWork) {
         for (StoragePoolWorkVO work : pendingWork) {
             try {
                 VMInstanceVO vmInstance = vmDao.findById(work.getVmId());
@@ -366,61 +176,188 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation {
                     continue;
                 }
 
-                // if the instance is of type consoleproxy, call the console
+                switch (vmInstance.getType()) {
-                // proxy
+                    case ConsoleProxy:
-                if (vmInstance.getType().equals(VirtualMachine.Type.ConsoleProxy)) {
+                    case SecondaryStorageVm:
-
+                    case DomainRouter:
-                    ConsoleProxyVO consoleProxy = _consoleProxyDao
+                        handleVmStart(work, vmInstance);
-                            .findById(vmInstance.getId());
+                        break;
-                    vmMgr.advanceStart(consoleProxy.getUuid(), null, null);
+                    case User:
-                    // update work queue
+                        handleUserVmStart(work, vmInstance);
-                    work.setStartedAfterMaintenance(true);
+                        break;
-                    _storagePoolWorkDao.update(work.getId(), work);
-                }
-
-                // if the instance is of type ssvm, call the ssvm manager
-                if (vmInstance.getType().equals(
-                        VirtualMachine.Type.SecondaryStorageVm)) {
-                    SecondaryStorageVmVO ssVm = _secStrgDao.findById(vmInstance
-                            .getId());
-                    vmMgr.advanceStart(ssVm.getUuid(), null, null);
-
-                    // update work queue
-                    work.setStartedAfterMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-                }
-
-                // if the instance is of type domain router vm, call the network
-                // manager
-                if (vmInstance.getType().equals(VirtualMachine.Type.DomainRouter)) {
-                    DomainRouterVO domR = _domrDao.findById(vmInstance.getId());
-                    vmMgr.advanceStart(domR.getUuid(), null, null);
-                    // update work queue
-                    work.setStartedAfterMaintenance(true);
-                    _storagePoolWorkDao.update(work.getId(), work);
-                }
-
-                // if the instance is of type user vm, call the user vm manager
-                if (vmInstance.getType().equals(VirtualMachine.Type.User)) {
-                    // check if the vm has a root volume. If not, remove the item from the queue, the vm should be
-                    // started only when it has at least one root volume attached to it
-                    // don't allow to start vm that doesn't have a root volume
-                    if (volumeDao.findByInstanceAndType(vmInstance.getId(), Volume.Type.ROOT).isEmpty()) {
-                        _storagePoolWorkDao.remove(work.getId());
-                    } else {
-                        UserVmVO userVm = userVmDao.findById(vmInstance.getId());
-
-                        vmMgr.advanceStart(userVm.getUuid(), null, null);
-                        work.setStartedAfterMaintenance(true);
-                        _storagePoolWorkDao.update(work.getId(), work);
-                    }
                 }
             } catch (Exception e) {
                 logger.debug("Failed start vm", e);
                 throw new CloudRuntimeException(e.toString());
             }
         }
-        return false;
     }
 
+    private void handleStopVmForMigration(StoragePoolWorkVO work, VMInstanceVO vmInstance) throws ResourceUnavailableException, OperationTimedoutException {
+        vmMgr.advanceStop(vmInstance.getUuid(), false);
+        // update work status
+        work.setStoppedForMaintenance(true);
+        _storagePoolWorkDao.update(work.getId(), work);
+    }
+
+    private void handleVmMigration(boolean restart, StoragePoolWorkVO work, VMInstanceVO vmInstance) throws ResourceUnavailableException, OperationTimedoutException, InsufficientCapacityException {
+        handleStopVmForMigration(work, vmInstance);
+
+        if (restart) {
+            handleVmStart(work, vmInstance);
+        }
+    }
+    private void handleVmStart(StoragePoolWorkVO work, VMInstanceVO vmInstance) throws InsufficientCapacityException, ResourceUnavailableException, OperationTimedoutException {
+        vmMgr.advanceStart(vmInstance.getUuid(), null, null);
+        // update work queue
+        work.setStartedAfterMaintenance(true);
+        _storagePoolWorkDao.update(work.getId(), work);
+    }
+
+    private void enqueueMigrationsForVolumes(List<VolumeVO> allVolumes, StoragePoolVO pool) {
+        for (VolumeVO volume : allVolumes) {
+            VMInstanceVO vmInstance = vmDao.findById(volume.getInstanceId());
+
+            if (vmInstance == null) {
+                continue;
+            }
+
+            // enqueue sp work
+            if (vmInstance.getState().equals(State.Running) || vmInstance.getState().equals(State.Starting) || vmInstance.getState().equals(State.Stopping)) {
+
+                try {
+                    StoragePoolWorkVO work = new StoragePoolWorkVO(vmInstance.getId(), pool.getId(), false, false, server.getId());
+                    _storagePoolWorkDao.persist(work);
+                } catch (Exception e) {
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Work record already exists, re-using by re-setting values");
+                    }
+                    StoragePoolWorkVO work = _storagePoolWorkDao.findByPoolIdAndVmId(pool.getId(), vmInstance.getId());
+                    work.setStartedAfterMaintenance(false);
+                    work.setStoppedForMaintenance(false);
+                    work.setManagementServerId(server.getId());
+                    _storagePoolWorkDao.update(work.getId(), work);
+                }
+            }
+        }
+    }
+
+    private void removeHeartbeatForHostsFromPool(List<HostVO> hosts, StoragePool storagePool) {
+        // remove heartbeat
+        for (HostVO host : hosts) {
+            ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(false, storagePool);
+            final Answer answer = agentMgr.easySend(host.getId(), cmd);
+            if (answer == null || !answer.getResult()) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("ModifyStoragePool false failed due to {}", ((answer == null) ? "answer null" : answer.getDetails()));
+                }
+            } else {
+                reportSucceededModifyStorePool(storagePool, (ModifyStoragePoolAnswer) answer, host, false);
+            }
+        }
+    }
+
+    private void reportSucceededModifyStorePool(StoragePool storagePool, ModifyStoragePoolAnswer answer, HostVO host, boolean add) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("ModifyStoragePool succeeded for {}", add ? "adding" : "removing");
+        }
+        if (storagePool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) {
+            logger.debug("Started synchronising datastore cluster storage pool {} with vCenter", storagePool);
+            storageManager.syncDatastoreClusterStoragePool(storagePool.getId(), answer.getDatastoreClusterChildren(), host.getId());
+        }
+    }
+
+    /**
+     * Handling the Zone wide and cluster wide primary storage
+     * if the storage scope is ZONE wide, then get all the hosts for which hypervisor ZoneWideStoragePools created to send ModifyStoragePoolCommand
+     * TODO: if it's zone wide, this code will list a lot of hosts in the zone, which may cause performance/OOM issue.
+     * @param pool pool to check for connected hosts
+     * @return a list of connected hosts
+     */
+    private List<HostVO> getHostsForStoragePool(StoragePoolVO pool) {
+        List<HostVO> hosts;
+        if (pool.getScope().equals(ScopeType.ZONE)) {
+            if (HypervisorType.Any.equals(pool.getHypervisor())) {
+                hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZone(pool.getDataCenterId());
+            }
+            else {
+                hosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(pool.getHypervisor(), pool.getDataCenterId());
+            }
+        } else {
+            hosts = _resourceMgr.listHostsInClusterByStatus(pool.getClusterId(), Status.Up);
+        }
+        return hosts;
+    }
+
+    /**
+     * Handling Zone and Cluster wide storage scopes. Depending on the scope of the pool, check for other storage pools in the same scope
+     * If the storage is ZONE wide then we pass podId and cluster id as null as they will be empty for Zone wide storage
+     *
+     * @param pool pool to check for other pools in the same scope
+     */
+    private void getStoragePoolForSpecification(StoragePoolVO pool) {
+        List<StoragePoolVO> storagePools;
+        if (pool.getScope() == ScopeType.ZONE) {
+            storagePools = primaryDataStoreDao.listBy(pool.getDataCenterId(), null, null, ScopeType.ZONE);
+        } else {
+            storagePools = primaryDataStoreDao.listBy(pool.getDataCenterId(), pool.getPodId(), pool.getClusterId(), ScopeType.CLUSTER);
+        }
+        checkHierarchyForPreparingForMaintenance(pool, storagePools);
+    }
+
+    /**
+     * If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode
+     * @param pool target to put in maintenance
+     * @param storagePools list of possible peers/parents/children
+     */
+    private static void checkHierarchyForPreparingForMaintenance(StoragePoolVO pool, List<StoragePoolVO> storagePools) {
+        for (StoragePoolVO storagePool : storagePools) {
+            if (!(storagePool.getParent().equals(pool.getParent()) || !pool.getParent().equals(storagePool.getId())) &&
+                (storagePool.getStatus() == StoragePoolStatus.PrepareForMaintenance)) {
+                    throw new CloudRuntimeException(String.format("Only one storage pool in a cluster can be in PrepareForMaintenance mode, %s is already in  PrepareForMaintenance mode ", storagePool));
+            }
+        }
+    }
+
+    /**
+     *         // check if the vm has a root volume. If not, remove the item from the queue, the vm should be
+     *         // started only when it has at least one root volume attached to it
+     *         // don't allow to start vm that doesn't have a root volume
+     * @param work work item to handle for this VM
+     * @param vmInstance VM to start
+     * @throws InsufficientCapacityException no migration target found
+     * @throws ResourceUnavailableException a resource required for migration is not in the expected state
+     * @throws OperationTimedoutException migration operation took too long
+     */
+    private void handleUserVmStart(StoragePoolWorkVO work, VMInstanceVO vmInstance) throws InsufficientCapacityException, ResourceUnavailableException, OperationTimedoutException {
+        if (volumeDao.findByInstanceAndType(vmInstance.getId(), Volume.Type.ROOT).isEmpty()) {
+            _storagePoolWorkDao.remove(work.getId());
+        } else {
+            handleVmStart(work, vmInstance);
+        }
+    }
+
+    private void addHeartbeatToHostsInPool(List<HostVO> hosts, StoragePool pool, Pair<Map<String, String>, Boolean> nfsMountOpts) {
+        for (HostVO host : hosts) {
+            ModifyStoragePoolCommand msPoolCmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
+            final Answer answer = agentMgr.easySend(host.getId(), msPoolCmd);
+            if (answer == null || !answer.getResult()) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("ModifyStoragePool add failed due to {}", ((answer == null) ? "answer null" : answer.getDetails()));
+                }
+                if (answer != null && nfsMountOpts.second()) {
+                    logger.error("Unable to attach storage pool to the host {} due to {}",  host,  answer.getDetails());
+                    StringBuilder exceptionSB = new StringBuilder("Unable to attach storage pool to the host ").append(host.getName());
+                    String reason = storageManager.getStoragePoolMountFailureReason(answer.getDetails());
+                    if (reason!= null) {
+                        exceptionSB.append(". ").append(reason).append(".");
+                    }
+                    throw new CloudRuntimeException(exceptionSB.toString());
+                }
+            } else {
+                storageManager.updateStoragePoolHostVOAndBytes(pool, host.getId(), (ModifyStoragePoolAnswer) answer);
+                reportSucceededModifyStorePool(pool, (ModifyStoragePoolAnswer) answer, host, true);
+            }
+        }
+    }
 }