CLOUDSTACK-9782: New Background Polling Task Manager (#2218)

CloudStack has several background polling tasks that are spread across
the codebase, the aim of this work is to provide a single manager to
handle submission, execution and handling of background tasks. With
the framework implemented, existing oobm background task has been
refactored to use this manager.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java

@@ -74,7 +74,7 @@ public class ChangeOutOfBandManagementPasswordCmd extends BaseAsyncCmd {
         CallContext.current().setEventDetails("Host Id: " + host.getId() + " Password: " + getPassword().charAt(0) + "****");
         CallContext.current().putContextParameter(Host.class, host.getUuid());
 
-        final OutOfBandManagementResponse response = outOfBandManagementService.changeOutOfBandManagementPassword(host, getPassword());
+        final OutOfBandManagementResponse response = outOfBandManagementService.changePassword(host, getPassword());
         response.setResponseName(getCommandName());
         setResponseObject(response);
     }

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java

@@ -83,7 +83,7 @@ public class ConfigureOutOfBandManagementCmd extends BaseCmd {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
         }
         CallContext.current().putContextParameter(Host.class, host.getUuid());
-        final OutOfBandManagementResponse response = outOfBandManagementService.configureOutOfBandManagement(host, getHostPMOptions());
+        final OutOfBandManagementResponse response = outOfBandManagementService.configure(host, getHostPMOptions());
         response.setId(host.getUuid());
         response.setResponseName(getCommandName());
         setResponseObject(response);

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java

@@ -80,7 +80,7 @@ public class IssueOutOfBandManagementPowerActionCmd extends BaseAsyncCmd {
         CallContext.current().setEventDetails("Host Id: " + host.getId() + " Action: " + powerOperation.toString());
         CallContext.current().putContextParameter(Host.class, host.getUuid());
 
-        final OutOfBandManagementResponse response = outOfBandManagementService.executeOutOfBandManagementPowerOperation(host, powerOperation, getActionTimeout());
+        final OutOfBandManagementResponse response = outOfBandManagementService.executePowerOperation(host, powerOperation, getActionTimeout());
         response.setResponseName(getCommandName());
         setResponseObject(response);
     }

api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java

@@ -30,9 +30,6 @@ public interface OutOfBandManagementService {
     ConfigKey<Long> ActionTimeout = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.action.timeout", "60",
                     "The out of band management action timeout in seconds, configurable by cluster", true, ConfigKey.Scope.Cluster);
 
-    ConfigKey<Long> SyncThreadInterval = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.sync.interval", "300000",
-            "The interval (in milliseconds) when the out-of-band management background sync are retrieved", true, ConfigKey.Scope.Global);
-
     ConfigKey<Integer> SyncThreadPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "outofbandmanagement.sync.poolsize", "50",
             "The out of band management background sync thread pool size", true, ConfigKey.Scope.Global);
 
@@ -49,7 +46,7 @@ public interface OutOfBandManagementService {
     OutOfBandManagementResponse disableOutOfBandManagement(Cluster cluster);
     OutOfBandManagementResponse disableOutOfBandManagement(Host host);
 
-    OutOfBandManagementResponse configureOutOfBandManagement(Host host, ImmutableMap<OutOfBandManagement.Option, String> options);
+    OutOfBandManagementResponse configure(Host host, ImmutableMap<OutOfBandManagement.Option, String> options);
-    OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout);
+    OutOfBandManagementResponse executePowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout);
-    OutOfBandManagementResponse changeOutOfBandManagementPassword(Host host, String password);
+    OutOfBandManagementResponse changePassword(Host host, String password);
 }

api/src/org/apache/cloudstack/poll/BackgroundPollManager.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+public interface BackgroundPollManager {
+    /**
+     * Submits a background poll task that need to run continuously in the background
+     * to poll external resources, update states, trigger actions etc.
+     * Tasks must be submitted by a manager in configure-phase, the list of submitted tasks
+     * are then submitted to the internal executor service during start-phase.
+     * @param task periodic background task
+     * @since 4.11
+     */
+    void submitTask(final BackgroundPollTask task);
+}

api/src/org/apache/cloudstack/poll/BackgroundPollTask.java

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+public interface BackgroundPollTask extends Runnable {
+}

developer/developer-prefill.sql

@@ -114,10 +114,6 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
             VALUES ('Advanced', 'DEFAULT', 'management-server',
             'ping.timeout', '1.5');
 
-INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
-            VALUES ('Advanced', 'DEFAULT', 'management-server',
-            'outofbandmanagement.sync.interval', '1000');
-
 -- Enable dynamic RBAC by default for fresh deployments
 INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
             VALUES ('Advanced', 'DEFAULT', 'RoleService',

engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -49,6 +49,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
@@ -120,6 +121,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     ConfigurationDao _configDao;
     @Inject
     ConfigDepot _configDepot;
+    @Inject
+    private OutOfBandManagementDao outOfBandManagementDao;
 
     protected ClusteredAgentManagerImpl() {
         super();
@@ -736,7 +739,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid());
             final long lastPing = (System.currentTimeMillis() >> 10) - getTimeout();
             _hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing);
-            outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(vo.getMsid());
+            outOfBandManagementDao.expireServerOwnership(vo.getMsid());
             s_logger.info("Deleting entries from op_host_transfer table for Management server " + vo.getMsid());
             cleanupTransferMap(vo.getMsid());
         }

engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java

@@ -27,5 +27,5 @@ import java.util.List;
 public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> {
     OutOfBandManagement findByHost(long hostId);
     List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
-    void expireOutOfBandManagementOwnershipByServer(long serverId);
+    void expireServerOwnership(long serverId);
 }

engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java

@@ -110,7 +110,7 @@ public class OutOfBandManagementDaoImpl extends GenericDaoBase<OutOfBandManageme
     }
 
     @Override
-    public void expireOutOfBandManagementOwnershipByServer(long serverId) {
+    public void expireServerOwnership(long serverId) {
         final String resetOwnerSql = "UPDATE oobm set mgmt_server_id=NULL, power_state=NULL where mgmt_server_id=?";
         executeExpireOwnershipSql(resetOwnerSql, serverId);
         if (LOG.isDebugEnabled()) {

server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml

@@ -70,6 +70,10 @@
             value="#{resourceDiscoverersRegistry.registered}" />
     </bean>
 
+    <!-- the new background poll manager -->
+    <bean id="bgPollManager" class="org.apache.cloudstack.poll.BackgroundPollManagerImpl">
+    </bean>
+
     <bean id="highAvailabilityManagerExtImpl" class="com.cloud.ha.HighAvailabilityManagerExtImpl">
         <property name="investigators" value="#{haInvestigatorsRegistry.registered}" />
         <property name="fenceBuilders" value="#{haFenceBuildersRegistry.registered}" />

server/src/com/cloud/server/StatsCollector.java

@@ -20,7 +20,6 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Calendar;
-import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -34,15 +33,6 @@ import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
-import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
-import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
-import org.apache.cloudstack.utils.identity.ManagementServerNode;
-import org.apache.cloudstack.utils.usage.UsageUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -56,6 +46,10 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.utils.graphite.GraphiteClient;
 import org.apache.cloudstack.utils.graphite.GraphiteException;
+import org.apache.cloudstack.utils.usage.UsageUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -177,8 +171,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     @Inject
     private HostDao _hostDao;
     @Inject
-    private OutOfBandManagementDao outOfBandManagementDao;
-    @Inject
     private UserVmDao _userVmDao;
     @Inject
     private VolumeDao _volsDao;
@@ -195,8 +187,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     @Inject
     private ResourceManager _resourceMgr;
     @Inject
-    private OutOfBandManagementService outOfBandManagementService;
-    @Inject
     private ConfigurationDao _configDao;
     @Inject
     private EndPointSelector _epSelector;
@@ -244,7 +234,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>();
 
     long hostStatsInterval = -1L;
-    long hostOutOfBandManagementStatsInterval = -1L;
     long hostAndVmStatsInterval = -1L;
     long storageStatsInterval = -1L;
     long volumeStatsInterval = -1L;
@@ -290,7 +279,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     private void init(Map<String, String> configs) {
         _executor = Executors.newScheduledThreadPool(6, new NamedThreadFactory("StatsCollector"));
 
-        hostOutOfBandManagementStatsInterval = OutOfBandManagementService.SyncThreadInterval.value();
         hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
         hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
         storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
@@ -337,10 +325,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             _executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS);
         }
 
-        if (hostOutOfBandManagementStatsInterval > 0) {
-            _executor.scheduleWithFixedDelay(new HostOutOfBandManagementStatsCollector(), 15000L, hostOutOfBandManagementStatsInterval, TimeUnit.MILLISECONDS);
-        }
-
         if (hostAndVmStatsInterval > 0) {
             _executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS);
         }
@@ -477,36 +461,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
         }
     }
 
-    class HostOutOfBandManagementStatsCollector extends ManagedContextRunnable {
-        @Override
-        protected void runInContext() {
-            try {
-                s_logger.trace("HostOutOfBandManagementStatsCollector is running...");
-                List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
-                if (outOfBandManagementHosts == null) {
-                    return;
-                }
-                for (OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) {
-                    Host host = _hostDao.findById(outOfBandManagementHost.getHostId());
-                    if (host == null) {
-                        continue;
-                    }
-                    if (outOfBandManagementService.isOutOfBandManagementEnabled(host)) {
-                        outOfBandManagementService.submitBackgroundPowerSyncTask(host);
-                    } else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
-                        if (outOfBandManagementService.transitionPowerStateToDisabled(Collections.singletonList(host))) {
-                            if (s_logger.isDebugEnabled()) {
-                                s_logger.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId());
-                            }
-                        }
-                    }
-                }
-            } catch (Throwable t) {
-                s_logger.error("Error trying to retrieve host out-of-band management stats", t);
-            }
-        }
-    }
-
     class VmStatsCollector extends ManagedContextRunnable {
         @Override
         protected void runInContext() {

server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java

@@ -44,10 +44,13 @@ import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
 import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
 import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.cloudstack.poll.BackgroundPollManager;
+import org.apache.cloudstack.poll.BackgroundPollTask;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -78,6 +81,8 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
     private HostDao hostDao;
     @Inject
     private AlertManager alertMgr;
+    @Inject
+    private BackgroundPollManager backgroundPollManager;
 
     private String name;
     private long serviceId;
@@ -202,12 +207,14 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
         OutOfBandManagement.PowerState currentPowerState = outOfBandManagementHost.getPowerState();
         try {
             OutOfBandManagement.PowerState newPowerState = OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState, event);
-            boolean result = outOfBandManagementDao.updateState(currentPowerState, event, newPowerState, outOfBandManagementHost, null);
+            boolean result = OutOfBandManagement.PowerState.getStateMachine().transitTo(outOfBandManagementHost, event, null, outOfBandManagementDao);
             if (result) {
                 final String message = String.format("Transitioned out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId());
                 LOG.debug(message);
-                ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
+                if (newPowerState == OutOfBandManagement.PowerState.Unknown) {
-                        EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
+                    ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
+                            EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
+                }
             }
             return result;
         } catch (NoTransitionException ignored) {
@@ -278,7 +285,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
 
     public void submitBackgroundPowerSyncTask(final Host host) {
         if (host != null) {
-            backgroundSyncBlockingExecutor.submit(new OutOfBandManagementBackgroundTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
+            backgroundSyncBlockingExecutor.submit(new PowerOperationTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
         }
     }
 
@@ -356,7 +363,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription = "updating out-of-band management configuration")
-    public OutOfBandManagementResponse configureOutOfBandManagement(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
+    public OutOfBandManagementResponse configure(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
         OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
         if (outOfBandManagementConfig == null) {
             outOfBandManagementConfig = outOfBandManagementDao.persist(new OutOfBandManagementVO(host.getId()));
@@ -384,7 +391,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing host out-of-band management action", async = true)
-    public OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
+    public OutOfBandManagementResponse executePowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
         checkOutOfBandManagementEnabledByZoneClusterHost(host);
         final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host);
         final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
@@ -428,7 +435,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription = "updating out-of-band management password")
-    public OutOfBandManagementResponse changeOutOfBandManagementPassword(final Host host, final String newPassword) {
+    public OutOfBandManagementResponse changePassword(final Host host, final String newPassword) {
         checkOutOfBandManagementEnabledByZoneClusterHost(host);
         if (Strings.isNullOrEmpty(newPassword)) {
             throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as provided new-password is null or empty for the host %s.", host.getUuid()));
@@ -503,7 +510,9 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
                 0L, TimeUnit.MILLISECONDS,
                 new ArrayBlockingQueue<Runnable>(10 * poolSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
 
-        LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize + " and background sync thread interval=" + SyncThreadInterval.value() + "s");
+        backgroundPollManager.submitTask(new OutOfBandManagementPowerStatePollTask());
+
+        LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize);
         return true;
     }
 
@@ -516,7 +525,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
     @Override
     public boolean stop() {
         backgroundSyncBlockingExecutor.shutdown();
-        outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(getId());
+        outOfBandManagementDao.expireServerOwnership(getId());
         return true;
     }
 
@@ -527,7 +536,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {ActionTimeout, SyncThreadInterval, SyncThreadPoolSize};
+        return new ConfigKey<?>[] {ActionTimeout, SyncThreadPoolSize};
     }
 
     public List<OutOfBandManagementDriver> getOutOfBandManagementDrivers() {
@@ -537,4 +546,36 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
     public void setOutOfBandManagementDrivers(List<OutOfBandManagementDriver> outOfBandManagementDrivers) {
         this.outOfBandManagementDrivers = outOfBandManagementDrivers;
     }
+
+    private final class OutOfBandManagementPowerStatePollTask extends ManagedContextRunnable implements BackgroundPollTask {
+        @Override
+        protected void runInContext() {
+            try {
+                if (LOG.isTraceEnabled()) {
+                    LOG.trace("Host out-of-band management power state poll task is running...");
+                }
+                final List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
+                if (outOfBandManagementHosts == null || outOfBandManagementHosts.isEmpty()) {
+                    return;
+                }
+                for (final OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) {
+                    final Host host = hostDao.findById(outOfBandManagementHost.getHostId());
+                    if (host == null) {
+                        continue;
+                    }
+                    if (isOutOfBandManagementEnabled(host)) {
+                        submitBackgroundPowerSyncTask(host);
+                    } else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
+                        if (transitionPowerStateToDisabled(Collections.singletonList(host))) {
+                            if (LOG.isDebugEnabled()) {
+                                LOG.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId());
+                            }
+                        }
+                    }
+                }
+            } catch (Throwable t) {
+                LOG.error("Error trying to retrieve host out-of-band management stats", t);
+            }
+        }
+    }
 }

server/src/org/apache/cloudstack/outofbandmanagement/PowerOperationTask.java

@@ -20,14 +20,14 @@ package org.apache.cloudstack.outofbandmanagement;
 import com.cloud.host.Host;
 import org.apache.log4j.Logger;
 
-public class OutOfBandManagementBackgroundTask implements Runnable {
+public class PowerOperationTask implements Runnable {
-    public static final Logger LOG = Logger.getLogger(OutOfBandManagementBackgroundTask.class);
+    public static final Logger LOG = Logger.getLogger(PowerOperationTask.class);
 
     final private OutOfBandManagementService service;
     final private Host host;
     final private OutOfBandManagement.PowerOperation powerOperation;
 
-    public OutOfBandManagementBackgroundTask(OutOfBandManagementService service, Host host, OutOfBandManagement.PowerOperation powerOperation) {
+    public PowerOperationTask(OutOfBandManagementService service, Host host, OutOfBandManagement.PowerOperation powerOperation) {
         this.service = service;
         this.host = host;
         this.powerOperation = powerOperation;
@@ -41,7 +41,7 @@ public class OutOfBandManagementBackgroundTask implements Runnable {
     @Override
     public void run() {
         try {
-            service.executeOutOfBandManagementPowerOperation(host, powerOperation, null);
+            service.executePowerOperation(host, powerOperation, null);
         } catch (Exception e) {
             LOG.warn(String.format("Out-of-band management background task operation=%s for host id=%d failed with: %s",
                     powerOperation.name(), host.getId(), e.getMessage()));

server/src/org/apache/cloudstack/poll/BackgroundPollManagerImpl.java

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+import com.cloud.utils.component.Manager;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Preconditions;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public final class BackgroundPollManagerImpl extends ManagerBase implements BackgroundPollManager, Manager {
+    public static final Logger LOG = Logger.getLogger(BackgroundPollManagerImpl.class);
+
+    private ScheduledExecutorService backgroundPollTaskScheduler;
+    private List<BackgroundPollTask> submittedTasks = new ArrayList<>();
+    private volatile boolean isConfiguredAndStarted = false;
+
+    public long getInitialDelay() {
+        return 5000L;
+    }
+
+    public long getRoundDelay() {
+        return 4000L;
+    }
+
+    @Override
+    public boolean start() {
+        if (isConfiguredAndStarted) {
+            return true;
+        }
+        backgroundPollTaskScheduler = Executors.newScheduledThreadPool(submittedTasks.size() + 1, new NamedThreadFactory("BackgroundTaskPollManager"));
+        for (final BackgroundPollTask task : submittedTasks) {
+            backgroundPollTaskScheduler.scheduleWithFixedDelay(task, getInitialDelay(), getRoundDelay(), TimeUnit.MILLISECONDS);
+            LOG.debug("Scheduled background poll task: " + task.getClass().getName());
+        }
+        isConfiguredAndStarted = true;
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        if (isConfiguredAndStarted) {
+            backgroundPollTaskScheduler.shutdown();
+        }
+        return true;
+    }
+
+    @Override
+    public void submitTask(final BackgroundPollTask task) {
+        Preconditions.checkNotNull(task);
+        if (isConfiguredAndStarted) {
+            throw new CloudRuntimeException("Background Poll Manager cannot accept poll task as it has been configured and started.");
+        }
+        LOG.debug("Background Poll Manager received task: " + task.getClass().getSimpleName());
+        submittedTasks.add(task);
+    }
+}

server/test/org/apache/cloudstack/poll/BackgroundPollManagerImplTest.java

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.poll;
+
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BackgroundPollManagerImplTest {
+
+    private BackgroundPollManagerImpl pollManager;
+    private DummyPollTask pollTask;
+
+    private class DummyPollTask extends ManagedContextRunnable implements BackgroundPollTask {
+        private boolean didIRun = false;
+        private long counter = 0;
+
+        public boolean didItRan() {
+            return didIRun;
+        }
+
+        public long getCounter() {
+            return counter;
+        }
+
+        @Override
+        protected void runInContext() {
+            didIRun = true;
+            counter++;
+        }
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        pollManager = new BackgroundPollManagerImpl();
+        pollTask = new DummyPollTask();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        pollManager.stop();
+    }
+
+    @Test
+    public void testSubmitValidTask() throws Exception {
+        Assert.assertFalse(pollTask.didItRan());
+        Assert.assertTrue(pollTask.getCounter() == 0);
+
+        pollManager.submitTask(pollTask);
+        pollManager.start();
+        Thread.sleep(pollManager.getInitialDelay()*2);
+
+        Assert.assertTrue(pollTask.didItRan());
+        Assert.assertTrue(pollTask.getCounter() > 0);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testSubmitNullTask() throws Exception {
+        pollManager.submitTask(null);
+    }
+
+}

setup/dev/advanced.cfg

@@ -113,6 +113,11 @@
                                     "username": "root",
                                     "url": "http://sim/c1/h0",
                                     "password": "password"
+                                },
+                                {
+                                    "username": "root",
+                                    "url": "http://sim/c1/h1",
+                                    "password": "password"
                                 }
                             ],
                             "clustertype": "CloudManaged",
@@ -120,6 +125,10 @@
                                 {
                                     "url": "nfs://10.147.28.6:/export/home/sandbox/primary2",
                                     "name": "PS2"
+                                },
+                                {
+                                    "url": "nfs://10.147.28.6:/export/home/sandbox/primary3",
+                                    "name": "PS2"
                                 }
                             ]
                         }
@@ -223,10 +232,6 @@
         {
             "name": "ping.timeout",
             "value": "1.5"
-        },
-        {
-            "name": "outofbandmanagement.sync.interval",
-            "value": "1000"
         }
     ],
     "mgtSvr": [

test/integration/smoke/test_outofbandmanagement.py

@@ -211,8 +211,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
         cmd.driver = 'randomDriverThatDoesNotExist'
         try:
             response = self.apiclient.configureOutOfBandManagement(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
 
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
@@ -239,20 +241,26 @@ class TestOutOfBandManagement(cloudstackTestCase):
         cmd.hostid = -1
         try:
             response = self.apiclient.enableOutOfBandManagementForHost(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
         try:
             cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
             response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
         try:
             cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
             response = self.apiclient.enableOutOfBandManagementForZone(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
 
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
@@ -265,20 +273,26 @@ class TestOutOfBandManagement(cloudstackTestCase):
         cmd.hostid = -1
         try:
             response = self.apiclient.disableOutOfBandManagementForHost(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
         try:
             cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd()
             response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
         try:
             cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd()
             response = self.apiclient.disableOutOfBandManagementForZone(cmd)
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
 
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
@@ -340,8 +354,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
 
         try:
             self.issuePowerActionCmd('STATUS')
-            self.fail("Exception was expected, oobm is disabled at zone level")
+        except Exception:
-        except Exception: pass
+            pass
+        else:
+            self.fail("Expected an exception to be thrown, failing")
 
         # Enable at zone level
         cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
@@ -350,18 +366,16 @@ class TestOutOfBandManagement(cloudstackTestCase):
 
         try:
             self.issuePowerActionCmd('STATUS')
-            self.fail("Exception was expected, oobm is disabled at cluster level")
+        except Exception:
-        except Exception: pass
+            pass
+        else:
+            self.fail("Expected an exception to be thrown, failing")
 
         # Check background thread syncs state to Disabled
         response = self.getHost(hostId=host.id).outofbandmanagement
         self.assertEqual(response.powerstate, 'Disabled')
         self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort())
-        interval = list_configurations(
+        self.checkSyncToState('Disabled', 2)
-            self.apiclient,
-            name='outofbandmanagement.sync.interval'
-        )[0].value
-        self.checkSyncToState('Disabled', interval)
 
         # Enable at cluster level
         cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
@@ -370,8 +384,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
 
         try:
             self.issuePowerActionCmd('STATUS')
-            self.fail("Exception was expected, oobm is disabled at host level")
+        except Exception:
-        except Exception: pass
+            pass
+        else:
+            self.fail("Expected an exception to be thrown, failing")
 
         # Enable at host level
         cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
@@ -452,25 +468,21 @@ class TestOutOfBandManagement(cloudstackTestCase):
             Tests out-of-band management background powerstate sync
         """
         self.debug("Testing oobm background sync")
-        interval = list_configurations(
-            self.apiclient,
-            name='outofbandmanagement.sync.interval'
-        )[0].value
 
         self.configureAndEnableOobm()
         bmc = IpmiServerContext().bmc
 
         bmc.powerstate = 'on'
-        self.checkSyncToState('On', interval)
+        self.checkSyncToState('On', 2)
 
         bmc.powerstate = 'off'
-        self.checkSyncToState('Off', interval)
+        self.checkSyncToState('Off', 2)
 
         # Check for unknown state (ipmi server not reachable)
         cmd = self.getOobmConfigCmd()
         cmd.port = 1
         response = self.apiclient.configureOutOfBandManagement(cmd)
-        self.checkSyncToState('Unknown', interval)
+        self.checkSyncToState('Unknown', 2)
 
 
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
@@ -532,15 +544,11 @@ class TestOutOfBandManagement(cloudstackTestCase):
             self.fail("Management server failed to expire ownership of fenced peer")
 
         self.debug("Testing oobm background sync should claim new ownership")
-        interval = list_configurations(
-            self.apiclient,
-            name='outofbandmanagement.sync.interval'
-        )[0].value
 
         bmc = IpmiServerContext().bmc
         bmc.powerstate = 'on'
 
-        self.checkSyncToState('On', interval)
+        self.checkSyncToState('On', 2)
 
         result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort()))
         newOwnerId = result[0][0]
@@ -585,8 +593,10 @@ class TestOutOfBandManagement(cloudstackTestCase):
 
         try:
             response = self.issuePowerActionCmd('STATUS')
+        except Exception:
+            pass
+        else:
             self.fail("Expected an exception to be thrown, failing")
-        except Exception: pass
 
         alerts = Alert.list(self.apiclient, keyword="auth-error",
                         listall=True)