CLOUDSTACK-9299: Out-of-band Management for CloudStack

Support access to a host’s out-of-band management interface (e.g. IPMI, iLO,
DRAC, etc.) to manage host power operations (on/off etc.) and querying current
power state in CloudStack.

Given the wide range of out-of-band management interfaces such as iLO and iDRA,
the service implementation allows for development of separate drivers as plugins.
This feature comes with a ipmitool based driver that uses the
ipmitool (http://linux.die.net/man/1/ipmitool) to communicate with any
out-of-band management interface that support IPMI 2.0.

This feature allows following common use-cases:
- Restarting stalled/failed hosts
- Powering off under-utilised hosts
- Powering on hosts for provisioning or to increase capacity
- Allowing system administrators to see the current power state of the host

For testing this feature `ipmisim` can be used:
https://pypi.python.org/pypi/ipmisim

FS:
https://cwiki.apache.org/confluence/display/CLOUDSTACK/Out-of-band+Management+for+CloudStack

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.travis.yml

@@ -38,7 +38,7 @@ env:
   matrix:
     - TESTS="smoke/test_affinity_groups smoke/test_affinity_groups_projects smoke/test_dynamicroles smoke/test_deploy_vgpu_enabled_vm smoke/test_deploy_vm_iso smoke/test_deploy_vm_root_resize smoke/test_deploy_vm_with_userdata smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_guest_vlan_range"
     - TESTS="smoke/test_hosts smoke/test_internal_lb smoke/test_iso smoke/test_list_ids_parameter smoke/test_loadbalance smoke/test_multipleips_per_nic smoke/test_network smoke/test_network_acl smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan"
-    - TESTS="smoke/test_over_provisioning smoke/test_password_server smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot smoke/test_resource_detail"
+    - TESTS="smoke/test_outofbandmanagement smoke/test_over_provisioning smoke/test_password_server smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot smoke/test_resource_detail"
     - TESTS="smoke/test_router_dhcphosts smoke/test_routers smoke/test_routers_iptables_default_policy smoke/test_routers_network_ops smoke/test_staticroles smoke/test_scale_vm smoke/test_secondary_storage smoke/test_service_offerings smoke/test_snapshots smoke/test_ssvm smoke/test_templates"
     - TESTS="smoke/test_usage_events smoke/test_vm_life_cycle smoke/test_vm_snapshots smoke/test_volumes smoke/test_vpc_redundant smoke/test_vpc_router_nics smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/misc/test_vm_ha smoke/misc/test_escalations_templates smoke/misc/test_vm_sync"
 

api/src/com/cloud/event/EventTypes.java

@@ -309,6 +309,14 @@ public class EventTypes {
     // Host
     public static final String EVENT_HOST_RECONNECT = "HOST.RECONNECT";
 
+    // Host Out-of-band management
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE = "HOST.OOBM.ENABLE";
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE = "HOST.OOBM.DISABLE";
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE = "HOST.OOBM.CONFIGURE";
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION = "HOST.OOBM.ACTION";
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD = "HOST.OOBM.CHANGEPASSWORD";
+    public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION = "HOST.OOBM.POWERSTATE.TRANSITION";
+
     // Maintenance
     public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL";
     public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS";
@@ -745,6 +753,14 @@ public class EventTypes {
         // Host
         entityEventDetails.put(EVENT_HOST_RECONNECT, Host.class);
 
+        // Host Out-of-band management
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE, Host.class);
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, Host.class);
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, Host.class);
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, Host.class);
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, Host.class);
+        entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, Host.class);
+
         // Maintenance
         entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class);
         entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class);

api/src/com/cloud/resource/ResourceService.java

@@ -18,6 +18,7 @@ package com.cloud.resource;
 
 import java.util.List;
 
+import com.cloud.dc.DataCenter;
 import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd;
 import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd;
 import org.apache.cloudstack.api.command.admin.host.AddHostCmd;
@@ -92,6 +93,8 @@ public interface ResourceService {
 
     Cluster getCluster(Long clusterId);
 
+    DataCenter getZone(Long zoneId);
+
     List<HypervisorType> getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId);
 
     boolean releaseHostReservation(Long hostId);

api/src/org/apache/cloudstack/alert/AlertService.java

@@ -66,6 +66,7 @@ public interface AlertService {
         public static final AlertType ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = new AlertType((short)26, "ALERT.RESOURCE.EXCEED", true);
         public static final AlertType ALERT_TYPE_SYNC = new AlertType((short)27, "ALERT.TYPE.SYNC", true);
         public static final AlertType ALERT_TYPE_UPLOAD_FAILED = new AlertType((short)28, "ALERT.UPLOAD.FAILED", true);
+        public static final AlertType ALERT_TYPE_OOBM_AUTH_ERROR = new AlertType((short)29, "ALERT.OOBM.AUTHERROR", true);
 
         public short getType() {
             return type;

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -21,6 +21,7 @@ public class ApiConstants {
     public static final String ACCOUNTS = "accounts";
     public static final String ACCOUNT_TYPE = "accounttype";
     public static final String ACCOUNT_ID = "accountid";
+    public static final String ADDRESS = "address";
     public static final String ALGORITHM = "algorithm";
     public static final String ALLOCATED_ONLY = "allocatedonly";
     public static final String API_KEY = "apikey";
@@ -76,6 +77,7 @@ public class ApiConstants {
     public static final String DEVICE_ID = "deviceid";
     public static final String DISK_OFFERING_ID = "diskofferingid";
     public static final String DISK_SIZE = "disksize";
+    public static final String DRIVER = "driver";
     public static final String ROOT_DISK_SIZE = "rootdisksize";
     public static final String DISPLAY_NAME = "displayname";
     public static final String DISPLAY_NETWORK = "displaynetwork";
@@ -177,6 +179,8 @@ public class ApiConstants {
     public static final String OS_TYPE_ID = "ostypeid";
     public static final String OS_DISPLAY_NAME = "osdisplayname";
     public static final String OS_NAME_FOR_HYPERVISOR = "osnameforhypervisor";
+    public static final String OUTOFBANDMANAGEMENT_POWERSTATE = "outofbandmanagementpowerstate";
+    public static final String OUTOFBANDMANAGEMENT_ENABLED = "outofbandmanagementenabled";
     public static final String PARAMS = "params";
     public static final String PARENT_DOMAIN_ID = "parentdomainid";
     public static final String PASSWORD = "password";
@@ -193,7 +197,7 @@ public class ApiConstants {
     public static final String PORTABLE_IP_ADDRESS = "portableipaddress";
     public static final String PORT_FORWARDING_SERVICE_ID = "portforwardingserviceid";
     public static final String POST_URL = "postURL";
-    public static final String PARENT = "parent";
+    public static final String POWER_STATE = "powerstate";
     public static final String PRIVATE_INTERFACE = "privateinterface";
     public static final String PRIVATE_IP = "privateip";
     public static final String PRIVATE_PORT = "privateport";

api/src/org/apache/cloudstack/api/BaseResponse.java

@@ -24,31 +24,6 @@ public abstract class BaseResponse implements ResponseObject {
     private transient String responseName;
     private transient String objectName;
 
-    @Override
-    public String getResponseName() {
-        return responseName;
-    }
-
-    @Override
-    public void setResponseName(String responseName) {
-        this.responseName = responseName;
-    }
-
-    @Override
-    public String getObjectName() {
-        return objectName;
-    }
-
-    @Override
-    public void setObjectName(String objectName) {
-        this.objectName = objectName;
-    }
-
-    @Override
-    public String getObjectId() {
-        return null;
-    }
-
     @SerializedName(ApiConstants.JOB_ID)
     @Param(description = "the UUID of the latest async job acting on this object")
     protected String jobId;
@@ -57,6 +32,38 @@ public abstract class BaseResponse implements ResponseObject {
     @Param(description = "the current status of the latest async job acting on this object")
     private Integer jobStatus;
 
+    public BaseResponse() {
+    }
+
+    public BaseResponse(final String objectName) {
+        this.objectName = objectName;
+    }
+
+    @Override
+    public final String getResponseName() {
+        return responseName;
+    }
+
+    @Override
+    public final void setResponseName(String responseName) {
+        this.responseName = responseName;
+    }
+
+    @Override
+    public final String getObjectName() {
+        return objectName;
+    }
+
+    @Override
+    public final void setObjectName(String objectName) {
+        this.objectName = objectName;
+    }
+
+    @Override
+    public String getObjectId() {
+        return null;
+    }
+
     @Override
     public String getJobId() {
         return jobId;

api/src/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java

@@ -81,9 +81,19 @@ public class ListHostsCmd extends BaseListCmd {
                description = "lists hosts in the same cluster as this VM and flag hosts with enough CPU/RAm to host this VM")
     private Long virtualMachineId;
 
+    @Parameter(name = ApiConstants.OUTOFBANDMANAGEMENT_ENABLED,
+            type = CommandType.BOOLEAN,
+            description = "list hosts for which out-of-band management is enabled")
+    private Boolean outOfBandManagementEnabled;
+
+    @Parameter(name = ApiConstants.OUTOFBANDMANAGEMENT_POWERSTATE,
+            type = CommandType.STRING,
+            description = "list hosts by its out-of-band management interface's power state. Its value can be one of [On, Off, Unknown]")
+    private String outOfBandManagementPowerState;
+
     @Parameter(name = ApiConstants.RESOURCE_STATE,
                type = CommandType.STRING,
-               description = "list hosts by resource state. Resource state represents current state determined by admin of host, valule can be one of [Enabled, Disabled, Unmanaged, PrepareForMaintenance, ErrorInMaintenance, Maintenance, Error]")
+               description = "list hosts by resource state. Resource state represents current state determined by admin of host, value can be one of [Enabled, Disabled, Unmanaged, PrepareForMaintenance, ErrorInMaintenance, Maintenance, Error]")
     private String resourceState;
 
     @Parameter(name = ApiConstants.DETAILS,
@@ -165,6 +175,15 @@ public class ListHostsCmd extends BaseListCmd {
         return resourceState;
     }
 
+
+    public Boolean isOutOfBandManagementEnabled() {
+        return outOfBandManagementEnabled;
+    }
+
+    public String getHostOutOfBandManagementPowerState() {
+        return outOfBandManagementPowerState;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ChangeOutOfBandManagementPasswordCmd.java

@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import com.google.common.base.Strings;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = ChangeOutOfBandManagementPasswordCmd.APINAME, description = "Changes out-of-band management interface password on the host and updates the interface configuration in CloudStack if the operation succeeds, else reverts the old password",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = true, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class ChangeOutOfBandManagementPasswordCmd extends BaseAsyncCmd {
+    public static final String APINAME = "changeOutOfBandManagementPassword";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, required = true,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the host")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "the new host management interface password of maximum length 16, if none is provided a random password would be used")
+    private String password;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Host host = _resourceService.getHost(getHostId());
+        if (host == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
+        }
+
+        CallContext.current().setEventDetails("Host Id: " + host.getId() + " Password: " + getPassword().charAt(0) + "****");
+        CallContext.current().putContextParameter(Host.class, host.getUuid());
+
+        final OutOfBandManagementResponse response = outOfBandManagementService.changeOutOfBandManagementPassword(host, getPassword());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public String getPassword() {
+        if (Strings.isNullOrEmpty(password)) {
+            password = _mgr.generateRandomPassword();
+        }
+        return password;
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "change out-of-band management password for host: " + getHostId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/ConfigureOutOfBandManagementCmd.java

@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = ConfigureOutOfBandManagementCmd.APINAME, description = "Configures a host's out-of-band management interface",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = true, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class ConfigureOutOfBandManagementCmd extends BaseCmd {
+    public static final String APINAME = "configureOutOfBandManagement";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, required = true,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the host")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.DRIVER, type = CommandType.STRING, required = true, description = "the host management interface driver, for example: ipmitool")
+    private String driver;
+
+    @Parameter(name = ApiConstants.ADDRESS, type = CommandType.STRING, required = true, description = "the host management interface IP address")
+    private String address;
+
+    @Parameter(name = ApiConstants.PORT, type = CommandType.STRING, required = true, description = "the host management interface port")
+    private String port;
+
+    @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "the host management interface user")
+    private String username;
+
+    @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, required = true, description = "the host management interface password")
+    private String password;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Host host = _resourceService.getHost(getHostId());
+        if (host == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
+        }
+        CallContext.current().putContextParameter(Host.class, host.getUuid());
+        final OutOfBandManagementResponse response = outOfBandManagementService.configureOutOfBandManagement(host, getHostPMOptions());
+        response.setId(host.getUuid());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public final ImmutableMap<OutOfBandManagement.Option, String> getHostPMOptions() {
+        final ImmutableMap.Builder<OutOfBandManagement.Option, String> builder = ImmutableMap.builder();
+        if (!Strings.isNullOrEmpty(driver)) {
+            builder.put(OutOfBandManagement.Option.DRIVER, driver);
+        }
+        if (!Strings.isNullOrEmpty(address)) {
+            builder.put(OutOfBandManagement.Option.ADDRESS, address);
+        }
+        if (!Strings.isNullOrEmpty(port)) {
+            builder.put(OutOfBandManagement.Option.PORT, port);
+        }
+        if (!Strings.isNullOrEmpty(username)) {
+            builder.put(OutOfBandManagement.Option.USERNAME, username);
+        }
+        if (!Strings.isNullOrEmpty(password)) {
+            builder.put(OutOfBandManagement.Option.PASSWORD, password);
+        }
+        return builder.build();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/DisableOutOfBandManagementForClusterCmd.java

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.org.Cluster;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = DisableOutOfBandManagementForClusterCmd.APINAME, description = "Disables out-of-band management for a cluster",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class DisableOutOfBandManagementForClusterCmd extends BaseAsyncCmd {
+    public static final String APINAME = "disableOutOfBandManagementForCluster";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.CLUSTER_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = ClusterResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the cluster")
+    private Long clusterId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Cluster cluster = _resourceService.getCluster(getClusterId());
+        if (cluster == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find cluster by ID: " + getClusterId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.disableOutOfBandManagement(cluster);
+
+        CallContext.current().setEventDetails("Cluster Id:" + cluster.getId() + " out-of-band management enabled: false");
+        CallContext.current().putContextParameter(Cluster.class, cluster.getUuid());
+
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getClusterId() {
+        return clusterId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "disable out-of-band management password for cluster: " + getClusterId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/DisableOutOfBandManagementForHostCmd.java

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = DisableOutOfBandManagementForHostCmd.APINAME, description = "Disables out-of-band management for a host",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class DisableOutOfBandManagementForHostCmd extends BaseAsyncCmd {
+    public static final String APINAME = "disableOutOfBandManagementForHost";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.HOST_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = HostResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the host")
+    private Long hostId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Host host = _resourceService.getHost(getHostId());
+        if (host == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.disableOutOfBandManagement(host);
+
+        CallContext.current().setEventDetails("Host Id:" + host.getId() + " out-of-band management enabled: false");
+        CallContext.current().putContextParameter(Host.class, host.getUuid());
+
+        response.setId(host.getUuid());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getHostId() {
+        return hostId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "disable out-of-band management password for host: " + getHostId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/DisableOutOfBandManagementForZoneCmd.java

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.dc.DataCenter;
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = DisableOutOfBandManagementForZoneCmd.APINAME, description = "Disables out-of-band management for a zone",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class DisableOutOfBandManagementForZoneCmd extends BaseAsyncCmd {
+    public static final String APINAME = "disableOutOfBandManagementForZone";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ZONE_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = ZoneResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the zone")
+    private Long zoneId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final DataCenter zone = _resourceService.getZone(getZoneId());
+        if (zone == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.disableOutOfBandManagement(zone);
+
+        CallContext.current().setEventDetails("Zone Id:" + zone.getId() + " out-of-band management enabled: false");
+        CallContext.current().putContextParameter(DataCenter.class, zone.getUuid());
+
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getZoneId() {
+        return zoneId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "disable out-of-band management password for zone: " + getZoneId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/EnableOutOfBandManagementForClusterCmd.java

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.org.Cluster;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = EnableOutOfBandManagementForClusterCmd.APINAME, description = "Enables out-of-band management for a cluster",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class EnableOutOfBandManagementForClusterCmd extends BaseAsyncCmd {
+    public static final String APINAME = "enableOutOfBandManagementForCluster";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.CLUSTER_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = ClusterResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the cluster")
+    private Long clusterId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Cluster cluster = _resourceService.getCluster(getClusterId());
+        if (cluster == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find cluster by ID: " + getClusterId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.enableOutOfBandManagement(cluster);
+
+        CallContext.current().setEventDetails("Cluster Id:" + cluster.getId() + " out-of-band management enabled: true");
+        CallContext.current().putContextParameter(Cluster.class, cluster.getUuid());
+
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getClusterId() {
+        return clusterId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "enable out-of-band management password for cluster: " + getClusterId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/EnableOutOfBandManagementForHostCmd.java

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = EnableOutOfBandManagementForHostCmd.APINAME, description = "Enables out-of-band management for a host",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class EnableOutOfBandManagementForHostCmd extends BaseAsyncCmd {
+    public static final String APINAME = "enableOutOfBandManagementForHost";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.HOST_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = HostResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the host")
+    private Long hostId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Host host = _resourceService.getHost(getHostId());
+        if (host == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.enableOutOfBandManagement(host);
+
+        CallContext.current().setEventDetails("Host Id:" + host.getId() + " out-of-band management enabled: true");
+        CallContext.current().putContextParameter(Host.class, host.getUuid());
+
+        response.setId(host.getUuid());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getHostId() {
+        return hostId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "enable out-of-band management password for host: " + getHostId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/EnableOutOfBandManagementForZoneCmd.java

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.dc.DataCenter;
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = EnableOutOfBandManagementForZoneCmd.APINAME, description = "Enables out-of-band management for a zone",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class EnableOutOfBandManagementForZoneCmd extends BaseAsyncCmd {
+    public static final String APINAME = "enableOutOfBandManagementForZone";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ZONE_ID, type = BaseCmd.CommandType.UUID, required = true, entityType = ZoneResponse.class,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the zone")
+    private Long zoneId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    final public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final DataCenter zone = _resourceService.getZone(getZoneId());
+        if (zone == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId());
+        }
+
+        OutOfBandManagementResponse response = outOfBandManagementService.enableOutOfBandManagement(zone);
+
+        CallContext.current().setEventDetails("Zone Id:" + zone.getId() + " out-of-band management enabled: true");
+        CallContext.current().putContextParameter(DataCenter.class, zone.getUuid());
+
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    final public Long getZoneId() {
+        return zoneId;
+    }
+
+    @Override
+    final public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "enable out-of-band management password for zone: " + getZoneId();
+    }
+}

api/src/org/apache/cloudstack/api/command/admin/outofbandmanagement/IssueOutOfBandManagementPowerActionCmd.java

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.outofbandmanagement;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiCommandJobType;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+
+import javax.inject.Inject;
+
+@APICommand(name = IssueOutOfBandManagementPowerActionCmd.APINAME, description = "Initiates the specified power action to the host's out-of-band management interface",
+        responseObject = OutOfBandManagementResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.9.0", authorized = {RoleType.Admin})
+public class IssueOutOfBandManagementPowerActionCmd extends BaseAsyncCmd {
+    public static final String APINAME = "issueOutOfBandManagementPowerAction";
+
+    @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, required = true,
+            validations = {ApiArgValidator.PositiveNumber}, description = "the ID of the host")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.TIMEOUT, type = CommandType.LONG, description = "optional operation timeout in seconds that overrides the global or cluster-level out-of-band management timeout setting")
+    private Long actionTimeout;
+
+    @Parameter(name = ApiConstants.ACTION, type = CommandType.STRING, required = true,
+            validations = {ApiArgValidator.NotNullOrEmpty}, description = "out-of-band management power actions, valid actions are: ON, OFF, CYCLE, RESET, SOFT, STATUS")
+    private String powerAction;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        final Host host = _resourceService.getHost(getHostId());
+        if (host == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
+        }
+        final PowerOperation powerOperation = PowerOperation.valueOf(getPowerAction());
+
+        CallContext.current().setEventDetails("Host Id: " + host.getId() + " Action: " + powerOperation.toString());
+        CallContext.current().putContextParameter(Host.class, host.getUuid());
+
+        final OutOfBandManagementResponse response = outOfBandManagementService.executeOutOfBandManagementPowerOperation(host, powerOperation, getActionTimeout());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public Long getActionTimeout() {
+        return actionTimeout;
+    }
+
+    public String getPowerAction() {
+        return powerAction;
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "issue out-out-band management power action: " + getPowerAction() + " on host: " + getHostId();
+    }
+
+    @Override
+    public ApiCommandJobType getInstanceType() {
+        return ApiCommandJobType.Host;
+    }
+}

api/src/org/apache/cloudstack/api/response/ClusterResponse.java

@@ -17,7 +17,9 @@
 package org.apache.cloudstack.api.response;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.google.gson.annotations.SerializedName;
 
@@ -86,6 +88,10 @@ public class ClusterResponse extends BaseResponse {
     @Param(description = "Ovm3 VIP to use for pooling and/or clustering")
     private String ovm3vip;
 
+    @SerializedName(ApiConstants.RESOURCE_DETAILS)
+    @Param(description = "Meta data associated with the zone (key/value pairs)")
+    private Map<String, String> resourceDetails;
+
     public String getId() {
         return id;
     }
@@ -197,4 +203,11 @@ public class ClusterResponse extends BaseResponse {
     public String getOvm3Vip() {
         return ovm3vip;
     }
+
+    public void setResourceDetails(Map<String, String> details) {
+        if (details == null) {
+            return;
+        }
+        this.resourceDetails = new HashMap<>(details);
+    }
 }

api/src/org/apache/cloudstack/api/response/HostResponse.java

@@ -24,6 +24,7 @@ import com.google.gson.annotations.SerializedName;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.EntityReference;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
 
 import java.util.Date;
 import java.util.HashMap;
@@ -200,6 +201,10 @@ public class HostResponse extends BaseResponse {
     @Param(description = "true if this host is suitable(has enough capacity and satisfies all conditions like hosttags, max guests vm limit etc) to migrate a VM to it , false otherwise")
     private Boolean suitableForMigration;
 
+    @SerializedName("outofbandmanagement")
+    @Param(description = "the host out-of-band management information")
+    private OutOfBandManagementResponse outOfBandManagementResponse;
+
     @SerializedName("resourcestate")
     @Param(description = "the resource state of the host")
     private String resourceState;
@@ -403,6 +408,14 @@ public class HostResponse extends BaseResponse {
         this.suitableForMigration = suitableForMigration;
     }
 
+    public OutOfBandManagementResponse getOutOfBandManagementResponse() {
+        return outOfBandManagementResponse;
+    }
+
+    public void setOutOfBandManagementResponse(final OutOfBandManagement outOfBandManagementConfig) {
+        this.outOfBandManagementResponse =  new OutOfBandManagementResponse(outOfBandManagementConfig);
+    }
+
     public String getResourceState() {
         return resourceState;
     }

api/src/org/apache/cloudstack/api/response/OutOfBandManagementResponse.java

@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.host.Host;
+import com.cloud.serializer.Param;
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.EntityReference;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+
+@EntityReference(value = Host.class)
+public class OutOfBandManagementResponse extends BaseResponse {
+    @SerializedName(ApiConstants.HOST_ID)
+    @Param(description = "the ID of the host")
+    private String id;
+
+    @SerializedName(ApiConstants.POWER_STATE)
+    @Param(description = "the out-of-band management interface powerState of the host")
+    private OutOfBandManagement.PowerState powerState;
+
+    @SerializedName(ApiConstants.ENABLED)
+    @Param(description = "true if out-of-band management is enabled for the host")
+    private Boolean enabled;
+
+    @SerializedName(ApiConstants.DRIVER)
+    @Param(description = "the out-of-band management driver for the host")
+    private String driver;
+
+    @SerializedName(ApiConstants.ADDRESS)
+    @Param(description = "the out-of-band management interface address")
+    private String ipAddress;
+
+    @SerializedName(ApiConstants.PORT)
+    @Param(description = "the out-of-band management interface port")
+    private String port;
+
+    @SerializedName(ApiConstants.USERNAME)
+    @Param(description = "the out-of-band management interface username")
+    private String username;
+
+    @SerializedName(ApiConstants.PASSWORD)
+    @Param(description = "the out-of-band management interface password")
+    private String password;
+
+    @SerializedName(ApiConstants.ACTION)
+    @Param(description = "the out-of-band management action (if issued)")
+    private String outOfBandManagementAction;
+
+    @SerializedName(ApiConstants.DESCRIPTION)
+    @Param(description = "the operation result description")
+    private String resultDescription;
+
+    @SerializedName(ApiConstants.STATUS)
+    @Param(description = "the operation result")
+    private Boolean success;
+
+    public OutOfBandManagementResponse() {
+        super("outofbandmanagement");
+    }
+
+    public OutOfBandManagementResponse(final OutOfBandManagement outOfBandManagementConfig) {
+        this();
+        if (outOfBandManagementConfig == null) {
+            this.setEnabled(false);
+            this.setPowerState(OutOfBandManagement.PowerState.Disabled);
+            return;
+        }
+        this.setEnabled(outOfBandManagementConfig.isEnabled());
+        if (outOfBandManagementConfig.getPowerState() != null) {
+            this.setPowerState(outOfBandManagementConfig.getPowerState());
+        } else {
+            this.setPowerState(OutOfBandManagement.PowerState.Unknown);
+        }
+        this.setDriver(outOfBandManagementConfig.getDriver());
+        this.setIpAddress(outOfBandManagementConfig.getAddress());
+        if (outOfBandManagementConfig.getPort() != null) {
+            this.setPort(String.valueOf(outOfBandManagementConfig.getPort()));
+        }
+        this.setUsername(outOfBandManagementConfig.getUsername());
+        if (!Strings.isNullOrEmpty(outOfBandManagementConfig.getPassword())) {
+            this.setPassword(outOfBandManagementConfig.getPassword().substring(0, 1) + "****");
+        }
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public OutOfBandManagement.PowerState getPowerState() {
+        return powerState;
+    }
+
+    public void setPowerState(OutOfBandManagement.PowerState powerState) {
+        this.powerState = powerState;
+    }
+
+    public Boolean getEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(Boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public String getDriver() {
+        return driver;
+    }
+
+    public void setDriver(String driver) {
+        this.driver = driver;
+    }
+
+    public String getIpAddress() {
+        return ipAddress;
+    }
+
+    public void setIpAddress(String ipAddress) {
+        this.ipAddress = ipAddress;
+    }
+
+    public String getPort() {
+        return port;
+    }
+
+    public void setPort(String port) {
+        this.port = port;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getOutOfBandManagementAction() {
+        return outOfBandManagementAction;
+    }
+
+    public void setOutOfBandManagementAction(String outOfBandManagementAction) {
+        this.outOfBandManagementAction = outOfBandManagementAction;
+    }
+
+    public String getResultDescription() {
+        return resultDescription;
+    }
+
+    public void setResultDescription(String resultDescription) {
+        this.resultDescription = resultDescription;
+    }
+
+    public Boolean getSuccess() {
+        return success;
+    }
+
+    public void setSuccess(Boolean success) {
+        this.success = success;
+    }
+}

api/src/org/apache/cloudstack/api/response/ZoneResponse.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -233,6 +234,9 @@ public class ZoneResponse extends BaseResponse {
     }
 
     public void setResourceDetails(Map<String, String> details) {
-        this.resourceDetails = details;
+        if (details == null) {
+            return;
+        }
+        this.resourceDetails = new HashMap<>(details);
     }
 }

api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagement.java

@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.utils.fsm.StateObject;
+import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
+
+import java.util.Set;
+
+public interface OutOfBandManagement extends StateObject<OutOfBandManagement.PowerState>, InternalIdentity {
+
+    PowerState getState();
+
+    Long getHostId();
+
+    boolean isEnabled();
+
+    PowerState getPowerState();
+
+    String getDriver();
+
+    String getAddress();
+
+    Integer getPort();
+
+    String getUsername();
+
+    String getPassword();
+
+    Long getManagementServerId();
+
+    void setEnabled(boolean enabled);
+
+    void setDriver(String driver);
+
+    void setAddress(String address);
+
+    void setPort(Integer port);
+
+    void setUsername(String username);
+
+    void setPassword(String password);
+
+    enum Option {
+        DRIVER,
+        ADDRESS,
+        PORT,
+        USERNAME,
+        PASSWORD
+    }
+
+    enum PowerOperation {
+        ON,
+        OFF,
+        CYCLE,
+        RESET,
+        SOFT,
+        STATUS,
+    }
+
+    enum PowerState {
+        On,
+        Off,
+        Unknown,
+        Disabled;
+
+        public enum Event {
+            On("Chassis Power is On"),
+            Off("Chassis Power is Off"),
+            AuthError("Authentication error happened"),
+            Unknown("An unknown error happened"),
+            Enabled("Out-of-band management enabled"),
+            Disabled("Out-of-band management disabled");
+
+            private String description;
+            Event(String description) {
+                this.description = description;
+            }
+            public String toString() {
+                return String.format("%s(%s)", super.toString(), this.getDescription());
+            }
+            public String getDescription() {
+                return description;
+            }
+            public Long getServerId() {
+                // TODO: change in future if we've better claim & ownership
+                // Right now the first one to update the db wins
+                // and mgmt server id would eventually become consistent
+                return ManagementServerNode.getManagementServerId();
+            }
+        }
+
+        public Event toEvent() {
+            if (this.equals(On)) {
+                return Event.On;
+            } else if (this.equals(Off)) {
+                return Event.Off;
+            } else if (this.equals(Disabled)) {
+                return Event.Disabled;
+            }
+            return Event.Unknown;
+        }
+
+        private static final StateMachine2<PowerState, Event, OutOfBandManagement> FSM = new StateMachine2<PowerState, Event, OutOfBandManagement>();
+        static {
+            FSM.addInitialTransition(Event.On, On);
+            FSM.addInitialTransition(Event.Off, Off);
+            FSM.addInitialTransition(Event.Unknown, Unknown);
+            FSM.addInitialTransition(Event.AuthError, Unknown);
+            FSM.addInitialTransition(Event.Disabled, Disabled);
+
+            FSM.addTransitionFromStates(Event.On, On, On, Off, Unknown, Disabled);
+            FSM.addTransitionFromStates(Event.Off, Off, On, Off, Unknown, Disabled);
+            FSM.addTransitionFromStates(Event.Unknown, Unknown, On, Off, Unknown, Disabled);
+            FSM.addTransitionFromStates(Event.AuthError, Unknown, On, Off, Disabled);
+            FSM.addTransitionFromStates(Event.Disabled, Disabled, On, Off, Unknown);
+        }
+
+        public static StateMachine2<PowerState, Event, OutOfBandManagement> getStateMachine() {
+            return FSM;
+        }
+
+        public PowerState getNextPowerState(Event e) throws NoTransitionException {
+            return FSM.getNextState(this, e);
+        }
+
+        public Set<Event> getPossibleEvents() {
+            return FSM.getPossibleEvents(this);
+        }
+
+    }
+}

api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementDriver.java

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.utils.component.Adapter;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+
+public interface OutOfBandManagementDriver extends Adapter {
+    OutOfBandManagementDriverResponse execute(OutOfBandManagementDriverCommand cmd);
+}

api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.dc.DataCenter;
+import com.cloud.host.Host;
+import com.cloud.org.Cluster;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.framework.config.ConfigKey;
+
+import java.util.List;
+
+public interface OutOfBandManagementService {
+
+    ConfigKey<Long> ActionTimeout = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.action.timeout", "60",
+                    "The out of band management action timeout in seconds, configurable by cluster", true, ConfigKey.Scope.Cluster);
+
+    ConfigKey<Long> SyncThreadInterval = new ConfigKey<Long>("Advanced", Long.class, "outofbandmanagement.sync.interval", "300000",
+            "The interval (in milliseconds) when the out-of-band management background sync are retrieved", true, ConfigKey.Scope.Global);
+
+    ConfigKey<Integer> SyncThreadPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "outofbandmanagement.sync.poolsize", "50",
+            "The out of band management background sync thread pool size", true, ConfigKey.Scope.Global);
+
+    long getId();
+    boolean isOutOfBandManagementEnabled(Host host);
+    void submitBackgroundPowerSyncTask(Host host);
+    boolean transitionPowerStateToDisabled(List<? extends Host> hosts);
+
+    OutOfBandManagementResponse enableOutOfBandManagement(DataCenter zone);
+    OutOfBandManagementResponse enableOutOfBandManagement(Cluster cluster);
+    OutOfBandManagementResponse enableOutOfBandManagement(Host host);
+
+    OutOfBandManagementResponse disableOutOfBandManagement(DataCenter zone);
+    OutOfBandManagementResponse disableOutOfBandManagement(Cluster cluster);
+    OutOfBandManagementResponse disableOutOfBandManagement(Host host);
+
+    OutOfBandManagementResponse configureOutOfBandManagement(Host host, ImmutableMap<OutOfBandManagement.Option, String> options);
+    OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(Host host, OutOfBandManagement.PowerOperation operation, Long timeout);
+    OutOfBandManagementResponse changeOutOfBandManagementPassword(Host host, String password);
+}

api/src/org/apache/cloudstack/outofbandmanagement/driver/OutOfBandManagementDriverChangePasswordCommand.java

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+
+public final class OutOfBandManagementDriverChangePasswordCommand extends OutOfBandManagementDriverCommand {
+    private final String newPassword;
+
+    public OutOfBandManagementDriverChangePasswordCommand(final ImmutableMap<OutOfBandManagement.Option, String> options, final Long timeout, final String newPassword) {
+        super(options, timeout);
+        this.newPassword = newPassword;
+    }
+
+    public final String getNewPassword() {
+        return newPassword;
+    }
+
+}

api/src/org/apache/cloudstack/outofbandmanagement/driver/OutOfBandManagementDriverCommand.java

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.joda.time.Duration;
+
+public abstract class OutOfBandManagementDriverCommand {
+    private final ImmutableMap<OutOfBandManagement.Option, String> options;
+    private final Duration timeout;
+
+    public OutOfBandManagementDriverCommand(final ImmutableMap<OutOfBandManagement.Option, String> options, final Long timeoutSeconds) {
+        this.options = options;
+        if (timeoutSeconds != null && timeoutSeconds > 0) {
+            this.timeout = new Duration(timeoutSeconds * 1000);
+        } else {
+            this.timeout = Duration.ZERO;
+        }
+    }
+
+    public final ImmutableMap<OutOfBandManagement.Option, String> getOptions() {
+        return options;
+    }
+
+    public final Duration getTimeout() {
+        return timeout;
+    }
+}

api/src/org/apache/cloudstack/outofbandmanagement/driver/OutOfBandManagementDriverPowerCommand.java

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+
+public final class OutOfBandManagementDriverPowerCommand extends OutOfBandManagementDriverCommand {
+    private final OutOfBandManagement.PowerOperation powerOperation;
+
+    public OutOfBandManagementDriverPowerCommand(final ImmutableMap<OutOfBandManagement.Option, String> options, final Long timeout, final OutOfBandManagement.PowerOperation powerOperation) {
+        super(options, timeout);
+        this.powerOperation = powerOperation;
+    }
+
+    public final OutOfBandManagement.PowerOperation getPowerOperation() {
+        return powerOperation;
+    }
+}

api/src/org/apache/cloudstack/outofbandmanagement/driver/OutOfBandManagementDriverResponse.java

@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver;
+
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+
+public class OutOfBandManagementDriverResponse {
+    private String result;
+    private String error;
+    private boolean success = false;
+    private boolean hasAuthFailure = false;
+    private OutOfBandManagement.PowerState powerState;
+
+    public OutOfBandManagementDriverResponse(String result, String error, boolean success) {
+        this.result = result;
+        this.error = error;
+        this.success = success;
+    }
+
+    public OutOfBandManagement.PowerState.Event toEvent() {
+        if (hasAuthFailure()) {
+            return OutOfBandManagement.PowerState.Event.AuthError;
+        }
+
+        if (!isSuccess() || powerState == null) {
+            return OutOfBandManagement.PowerState.Event.Unknown;
+        }
+
+        return powerState.toEvent();
+    }
+
+    public boolean isSuccess() {
+        return success;
+    }
+
+    public String getResult() {
+        return result;
+    }
+
+    public String getError() {
+        return error;
+    }
+
+    public void setSuccess(boolean success) {
+        this.success = success;
+    }
+
+    public void setResult(String result) {
+        this.result = result;
+    }
+
+    public void setError(String error) {
+        this.error = error;
+    }
+
+    public OutOfBandManagement.PowerState getPowerState() {
+        return powerState;
+    }
+
+    public void setPowerState(OutOfBandManagement.PowerState powerState) {
+        this.powerState = powerState;
+    }
+
+    public boolean hasAuthFailure() {
+        return hasAuthFailure;
+    }
+
+    public void setAuthFailure(boolean hasAuthFailure) {
+        this.hasAuthFailure = hasAuthFailure;
+    }
+}

client/WEB-INF/classes/resources/messages.properties

@@ -876,6 +876,7 @@ label.metrics.num.cpu.cores=Cores
 label.metrics.property=Property
 label.metrics.scope=Scope
 label.metrics.state=State
+label.metrics.outofbandmanagementpowerstate=Power State
 label.metrics.storagepool=Storage Pool
 label.metrics.vm.name=VM Name
 label.migrate.instance.to.host=Migrate instance to another host
@@ -1004,6 +1005,25 @@ label.pods=Pods
 label.port.forwarding.policies=Port forwarding policies
 label.port.forwarding=Port Forwarding
 label.port.range=Port Range
+label.powerstate=Power State
+label.outofbandmanagement=Out-of-band Management
+label.outofbandmanagement.action.issue=Issue Out-of-band Management Power Action
+label.outofbandmanagement.action=Action
+label.outofbandmanagement.address=Address
+label.outofbandmanagement.changepassword=Change Out-of-band Management Password
+label.outofbandmanagement.configure=Configure Out-of-band Management
+label.outofbandmanagement.driver=Driver
+label.outofbandmanagement.disable=Disable Out-of-band Management
+label.outofbandmanagement.enable=Enable Out-of-band Management
+label.outofbandmanagement.password=Password
+label.outofbandmanagement.port=Port
+label.outofbandmanagement.username=Username
+message.outofbandmanagement.changepassword=Change Out-of-band Management password
+message.outofbandmanagement.configure=Configure Out-of-band Management
+message.outofbandmanagement.disable=Disable Out-of-band Management
+message.outofbandmanagement.enable=Enable Out-of-band Management
+message.outofbandmanagement.issue=Issue Out-of-band Management Power Action
+message.outofbandmanagement.action.maintenance=Warning host is in maintenance mode
 label.PreSetup=PreSetup
 label.prev=Prev
 label.previous=Previous

client/pom.xml

@@ -241,6 +241,11 @@
       <artifactId>cloud-plugin-host-allocator-random</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-plugin-outofbandmanagement-driver-ipmitool</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-mom-rabbitmq</artifactId>

core/resources/META-INF/cloudstack/api/spring-core-lifecycle-api-context-inheritable.xml

@@ -61,6 +61,5 @@
         <property name="registry" ref="userPasswordEncodersRegistry" />
         <property name="typeClass" value="com.cloud.server.auth.UserAuthenticator" />
     </bean>
-    
 
 </beans>

core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml

@@ -306,5 +306,10 @@
         class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
         <property name="excludeKey" value="data.motion.strategies.exclude" />
     </bean>
-    
+
+    <bean id="outOfBandManagementDriversRegistry"
+          class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
+        <property name="orderConfigDefault" value="IPMITOOL" />
+    </bean>
+
 </beans>

core/resources/META-INF/cloudstack/outofbandmanagement/module.properties

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name=outofbandmanagement
+parent=core

core/resources/META-INF/cloudstack/outofbandmanagement/spring-core-lifecycle-outofbandmanagement-context-inheritable.xml

@@ -0,0 +1,36 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+                      http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd
+                      http://www.springframework.org/schema/context
+                      http://www.springframework.org/schema/context/spring-context-3.0.xsd"
+                      >
+
+    <bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
+        <property name="registry" ref="outOfBandManagementDriversRegistry" />
+        <property name="typeClass" value="org.apache.cloudstack.outofbandmanagement.OutOfBandManagementDriver" />
+    </bean>
+</beans>

debian/control

@@ -15,7 +15,7 @@ Description: A common package which contains files which are shared by several C
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6 | tomcat7, sudo, jsvc, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2
+Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6 | tomcat7, sudo, jsvc, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, ipmitool
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server

developer/developer-prefill.sql

@@ -106,6 +106,18 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
             VALUES ('Advanced', 'DEFAULT', 'management-server',
             'direct.agent.load.size', '1000');
 
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+            VALUES ('Advanced', 'DEFAULT', 'management-server',
+            'ping.interval', '10');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+            VALUES ('Advanced', 'DEFAULT', 'management-server',
+            'ping.timeout', '1.5');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+            VALUES ('Advanced', 'DEFAULT', 'management-server',
+            'outofbandmanagement.sync.interval', '2000');
+
 -- Enable dynamic RBAC by default for fresh deployments
 INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
             VALUES ('Advanced', 'DEFAULT', 'RoleService',

engine/orchestration/src/com/cloud/agent/manager/AgentManagerImpl.java

@@ -43,6 +43,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.framework.jobs.AsyncJob;
 import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.log4j.Logger;
 import org.slf4j.MDC;
@@ -139,6 +140,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     @Inject
     protected HostDao _hostDao = null;
     @Inject
+    protected OutOfBandManagementDao outOfBandManagementDao;
+    @Inject
     protected DataCenterDao _dcDao = null;
     @Inject
     protected HostPodDao _podDao = null;

engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -736,6 +736,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid());
             final long lastPing = (System.currentTimeMillis() >> 10) - getTimeout();
             _hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing);
+            outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(vo.getMsid());
             s_logger.info("Deleting entries from op_host_transfer table for Management server " + vo.getMsid());
             cleanupTransferMap(vo.getMsid());
         }

engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -347,6 +347,6 @@
   <bean id="databaseIntegrityChecker" class="com.cloud.upgrade.DatabaseIntegrityChecker" />
   <bean id="LBStickinessPolicyDetailsDaoImpl" class="org.apache.cloudstack.resourcedetail.dao.LBStickinessPolicyDetailsDaoImpl" />
   <bean id="LBHealthCheckPolicyDetailsDaoImpl" class="org.apache.cloudstack.resourcedetail.dao.LBHealthCheckPolicyDetailsDaoImpl" />
-
+  <bean id="outOfBandManagementDaoImpl" class="org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDaoImpl" />
 
 </beans>

engine/schema/src/com/cloud/dc/dao/DataCenterDetailsDao.java

@@ -16,10 +16,10 @@
 // under the License.
 package com.cloud.dc.dao;
 
-import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
-
 import com.cloud.dc.DataCenterDetailVO;
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
 
 public interface DataCenterDetailsDao extends GenericDao<DataCenterDetailVO, Long>, ResourceDetailsDao<DataCenterDetailVO> {
+    void persist(long zoneId, String name, String value);
 }

engine/schema/src/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java

@@ -16,7 +16,6 @@
 // under the License.
 package com.cloud.dc.dao;
 
-
 import org.apache.cloudstack.api.ResourceDetail;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
@@ -24,9 +23,21 @@ import org.apache.cloudstack.framework.config.ScopedConfigStorage;
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
 
 import com.cloud.dc.DataCenterDetailVO;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.TransactionLegacy;
 
 public class DataCenterDetailsDaoImpl extends ResourceDetailsDaoBase<DataCenterDetailVO> implements DataCenterDetailsDao, ScopedConfigStorage {
 
+    private final SearchBuilder<DataCenterDetailVO> DetailSearch;
+
+    DataCenterDetailsDaoImpl() {
+        DetailSearch = createSearchBuilder();
+        DetailSearch.and("zoneId", DetailSearch.entity().getResourceId(), SearchCriteria.Op.EQ);
+        DetailSearch.and("name", DetailSearch.entity().getName(), SearchCriteria.Op.EQ);
+        DetailSearch.done();
+    }
+
     @Override
     public Scope getScope() {
         return ConfigKey.Scope.Zone;
@@ -43,4 +54,17 @@ public class DataCenterDetailsDaoImpl extends ResourceDetailsDaoBase<DataCenterD
         super.addDetail(new DataCenterDetailVO(resourceId, key, value, display));
     }
 
+    @Override
+    public void persist(long zoneId, String name, String value) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+        SearchCriteria<DataCenterDetailVO> sc = DetailSearch.create();
+        sc.setParameters("zoneId", zoneId);
+        sc.setParameters("name", name);
+        expunge(sc);
+
+        DataCenterDetailVO vo = new DataCenterDetailVO(zoneId, name, value, true);
+        persist(vo);
+        txn.commit();
+    }
 }

engine/schema/src/com/cloud/host/dao/HostDao.java

@@ -81,6 +81,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
      */
     List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag);
 
+    List<HostVO> findByDataCenterId(Long zoneId);
+
     List<HostVO> findByPodId(Long podId);
 
     List<HostVO> findByClusterId(Long clusterId);
@@ -89,5 +91,7 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<Long> listAllHosts(long zoneId);
 
+    List<HostVO> listAllHostsByType(Host.Type type);
+
     HostVO findByPublicIp(String publicIp);
 }

engine/schema/src/com/cloud/host/dao/HostDaoImpl.java

@@ -1048,6 +1048,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return findOneBy(sc);
     }
 
+    @Override
+    public List<HostVO> findByDataCenterId(Long zoneId) {
+        SearchCriteria<HostVO> sc = DcSearch.create();
+        sc.setParameters("dc", zoneId);
+        sc.setParameters("type", Type.Routing);
+        return listBy(sc);
+    }
+
     @Override
     public List<HostVO> findByPodId(Long podId) {
         SearchCriteria<HostVO> sc = PodSearch.create();
@@ -1087,4 +1095,13 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId);
         return customSearch(sc, null);
     }
+
+    @Override
+    public List<HostVO> listAllHostsByType(Host.Type type) {
+        SearchCriteria<HostVO> sc = TypeSearch.create();
+        sc.setParameters("type", type);
+        sc.setParameters("resourceState", ResourceState.Enabled);
+
+        return listBy(sc);
+    }
 }

engine/schema/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementVO.java

@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.utils.db.Encrypt;
+import com.cloud.utils.db.StateMachine;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import java.util.Date;
+
+@Entity
+@Table(name = "oobm")
+public class OutOfBandManagementVO implements OutOfBandManagement {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private long id;
+
+    @Column(name = "host_id")
+    private Long hostId;
+
+    @Column(name = "enabled")
+    private boolean enabled = false;
+
+    // There is no setter for status because it has to be set in the dao code
+    @Enumerated(value = EnumType.STRING)
+    @StateMachine(state = PowerState.class, event = PowerState.Event.class)
+    @Column(name = "power_state", updatable = true, nullable = false, length = 32)
+    private PowerState powerState = null;
+
+    @Column(name = "driver")
+    private String driver;
+
+    @Column(name = "address")
+    private String address;
+
+    @Column(name = "port")
+    private Integer port;
+
+    @Column(name = "username")
+    private String username;
+
+    @Encrypt
+    @Column(name = "password")
+    private String password;
+
+    // This field should be updated every time the state is updated.
+    // There's no set method in the vo object because it is done with in the dao code.
+    @Column(name = "update_count", updatable = true, nullable = false)
+    protected long updateCount;
+
+    @Column(name = "update_time", updatable = true)
+    @Temporal(value = TemporalType.TIMESTAMP)
+    protected Date updateTime;
+
+    @Column(name = "mgmt_server_id")
+    private Long managementServerId;
+
+    public OutOfBandManagementVO(Long hostId) {
+        this.hostId = hostId;
+        this.powerState = PowerState.Disabled;
+    }
+
+    public OutOfBandManagementVO() {
+    }
+
+    @Override
+    public long getId() {
+        return id;
+    }
+
+    @Override
+    public PowerState getState() {
+        return powerState;
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public PowerState getPowerState() {
+        return powerState;
+    }
+
+    @Override
+    public String getDriver() {
+        return driver;
+    }
+
+    @Override
+    public String getAddress() {
+        return address;
+    }
+
+    @Override
+    public Integer getPort() {
+        return port;
+    }
+
+    @Override
+    public String getUsername() {
+        return username;
+    }
+
+    @Override
+    public String getPassword() {
+        return password;
+    }
+
+    public long incrUpdateCount() {
+        updateCount++;
+        return updateCount;
+    }
+
+    public long getUpdateCount() {
+        return updateCount;
+    }
+
+    public Date getUpdateTime() {
+        return updateTime;
+    }
+
+    @Override
+    public Long getManagementServerId() {
+        return managementServerId;
+    }
+
+    public void setHostId(Long hostId) {
+        this.hostId = hostId;
+    }
+
+    @Override
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    @Override
+    public void setDriver(String driver) {
+        this.driver = driver;
+    }
+
+    @Override
+    public void setAddress(String address) {
+        this.address = address;
+    }
+
+    @Override
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    @Override
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public void setManagementServerId(Long managementServerId) {
+        this.managementServerId = managementServerId;
+    }
+}

engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDao.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement.dao;
+
+import com.cloud.utils.db.GenericDao;
+import com.cloud.utils.fsm.StateDao;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
+
+import java.util.List;
+
+public interface OutOfBandManagementDao extends GenericDao<OutOfBandManagementVO, Long>, StateDao<OutOfBandManagement.PowerState, OutOfBandManagement.PowerState.Event, OutOfBandManagement> {
+    OutOfBandManagement findByHost(long hostId);
+    List<OutOfBandManagementVO> findAllByManagementServer(long serverId);
+    void expireOutOfBandManagementOwnershipByServer(long serverId);
+}

engine/schema/src/org/apache/cloudstack/outofbandmanagement/dao/OutOfBandManagementDaoImpl.java

@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement.dao;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.Attribute;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallbackNoReturn;
+import com.cloud.utils.db.TransactionLegacy;
+import com.cloud.utils.db.TransactionStatus;
+import com.cloud.utils.db.UpdateBuilder;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+
+@DB
+@Component
+public class OutOfBandManagementDaoImpl extends GenericDaoBase<OutOfBandManagementVO, Long> implements OutOfBandManagementDao {
+    private static final Logger LOG = Logger.getLogger(OutOfBandManagementDaoImpl.class);
+
+    private SearchBuilder<OutOfBandManagementVO> HostSearch;
+    private SearchBuilder<OutOfBandManagementVO> ManagementServerSearch;
+    private SearchBuilder<OutOfBandManagementVO> OutOfBandManagementOwnerSearch;
+    private SearchBuilder<OutOfBandManagementVO> StateUpdateSearch;
+
+    private Attribute PowerStateAttr;
+    private Attribute MsIdAttr;
+    private Attribute UpdateTimeAttr;
+
+    public OutOfBandManagementDaoImpl() {
+        super();
+
+        HostSearch = createSearchBuilder();
+        HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        HostSearch.done();
+
+        ManagementServerSearch = createSearchBuilder();
+        ManagementServerSearch.and("server", ManagementServerSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
+        ManagementServerSearch.done();
+
+        OutOfBandManagementOwnerSearch = createSearchBuilder();
+        OutOfBandManagementOwnerSearch.and("server", OutOfBandManagementOwnerSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
+        OutOfBandManagementOwnerSearch.or("serverNull", OutOfBandManagementOwnerSearch.entity().getManagementServerId(), SearchCriteria.Op.NULL);
+        OutOfBandManagementOwnerSearch.done();
+
+        StateUpdateSearch = createSearchBuilder();
+        StateUpdateSearch.and("status", StateUpdateSearch.entity().getPowerState(), SearchCriteria.Op.EQ);
+        StateUpdateSearch.and("id", StateUpdateSearch.entity().getId(), SearchCriteria.Op.EQ);
+        StateUpdateSearch.and("update", StateUpdateSearch.entity().getUpdateCount(), SearchCriteria.Op.EQ);
+        StateUpdateSearch.done();
+
+        PowerStateAttr = _allAttributes.get("powerState");
+        MsIdAttr = _allAttributes.get("managementServerId");
+        UpdateTimeAttr = _allAttributes.get("updateTime");
+        assert (PowerStateAttr != null && MsIdAttr != null && UpdateTimeAttr != null) : "Couldn't find one of these attributes";
+    }
+
+    @Override
+    public OutOfBandManagement findByHost(long hostId) {
+        SearchCriteria<OutOfBandManagementVO> sc = HostSearch.create("hostId", hostId);
+        return findOneBy(sc);
+    }
+
+    @Override
+    public List<OutOfBandManagementVO> findAllByManagementServer(long serverId) {
+        SearchCriteria<OutOfBandManagementVO> sc = OutOfBandManagementOwnerSearch.create();
+        sc.setParameters("server", serverId);
+        return listBy(sc, new Filter(OutOfBandManagementVO.class, "updateTime", true, null, null));
+    }
+
+    private void executeExpireOwnershipSql(final String sql, final long resource) {
+        Transaction.execute(new TransactionCallbackNoReturn() {
+            @Override
+            public void doInTransactionWithoutResult(TransactionStatus status) {
+                TransactionLegacy txn = TransactionLegacy.currentTxn();
+                try (final PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);) {
+                    pstmt.setLong(1, resource);
+                    pstmt.executeUpdate();
+                } catch (SQLException e) {
+                    txn.rollback();
+                    LOG.warn("Failed to expire ownership for out-of-band management server id: " + resource);
+                }
+            }
+        });
+    }
+
+    @Override
+    public void expireOutOfBandManagementOwnershipByServer(long serverId) {
+        final String resetOwnerSql = "UPDATE oobm set mgmt_server_id=NULL, power_state=NULL where mgmt_server_id=?";
+        executeExpireOwnershipSql(resetOwnerSql, serverId);
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Expired out-of-band management ownership for hosts owned by management server id:" + serverId);
+        }
+    }
+
+    @Override
+    public boolean updateState(OutOfBandManagement.PowerState oldStatus, OutOfBandManagement.PowerState.Event event, OutOfBandManagement.PowerState newStatus, OutOfBandManagement vo, Object data) {
+        OutOfBandManagementVO oobmHost = (OutOfBandManagementVO) vo;
+        if (oobmHost == null) {
+            if (LOG.isTraceEnabled()) {
+                LOG.trace("Invalid out-of-band management host view object provided");
+            }
+            return false;
+        }
+
+        Long newManagementServerId = event.getServerId();
+        // Avoid updates when old ownership and state are same as new
+        if (oldStatus == newStatus && (oobmHost.getManagementServerId() != null && oobmHost.getManagementServerId().equals(newManagementServerId))) {
+            return false;
+        }
+
+        if (event == OutOfBandManagement.PowerState.Event.Disabled) {
+            newManagementServerId = null;
+        }
+
+        SearchCriteria<OutOfBandManagementVO> sc = StateUpdateSearch.create();
+        sc.setParameters("status", oldStatus);
+        sc.setParameters("id", oobmHost.getId());
+        sc.setParameters("update", oobmHost.getUpdateCount());
+
+        oobmHost.incrUpdateCount();
+        UpdateBuilder ub = getUpdateBuilder(oobmHost);
+        ub.set(oobmHost, PowerStateAttr, newStatus);
+        ub.set(oobmHost, UpdateTimeAttr, DateUtil.currentGMTTime());
+        ub.set(oobmHost, MsIdAttr, newManagementServerId);
+
+        int result = update(ub, sc, null);
+        if (LOG.isDebugEnabled() && result <= 0) {
+            LOG.debug(String.format("Failed to update out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", oldStatus, newStatus, event, oobmHost.getHostId()));
+        }
+        return result > 0;
+    }
+}

plugins/outofbandmanagement-drivers/ipmitool/pom.xml

@@ -0,0 +1,41 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>cloud-plugin-outofbandmanagement-driver-ipmitool</artifactId>
+  <name>Apache CloudStack Plugin - Power Management Driver ipmitool</name>
+  <parent>
+    <groupId>org.apache.cloudstack</groupId>
+    <artifactId>cloudstack-plugins</artifactId>
+    <version>4.9.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+</project>

plugins/outofbandmanagement-drivers/ipmitool/resources/META-INF/cloudstack/ipmitool/module.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+name=ipmitool
+parent=outofbandmanagement

plugins/outofbandmanagement-drivers/ipmitool/resources/META-INF/cloudstack/ipmitool/spring-ipmitool-context.xml

@@ -0,0 +1,34 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+                      http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd
+                      http://www.springframework.org/schema/context
+                      http://www.springframework.org/schema/context/spring-context-3.0.xsd"
+                      >
+
+    <bean id="ipmitoolOutOfBandManagementDriver" class="org.apache.cloudstack.outofbandmanagement.driver.ipmitool.IpmitoolOutOfBandManagementDriver">
+        <property name="name" value="IPMITOOL" />
+    </bean>
+
+</beans>

plugins/outofbandmanagement-drivers/ipmitool/src/org/apache/cloudstack/outofbandmanagement/driver/ipmitool/IpmitoolOutOfBandManagementDriver.java

@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver.ipmitool;
+
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementDriver;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.log4j.Logger;
+import org.joda.time.Duration;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public final class IpmitoolOutOfBandManagementDriver extends AdapterBase implements OutOfBandManagementDriver, Configurable {
+    public static final Logger LOG = Logger.getLogger(IpmitoolOutOfBandManagementDriver.class);
+
+    private static volatile boolean isDriverEnabled = false;
+    private static boolean isIpmiToolBinAvailable = false;
+
+    private final ExecutorService ipmitoolExecutor = Executors.newFixedThreadPool(OutOfBandManagementService.SyncThreadPoolSize.value(), new NamedThreadFactory("IpmiToolDriver"));
+    private final IpmitoolWrapper IPMITOOL = new IpmitoolWrapper(ipmitoolExecutor);
+
+    public final ConfigKey<String> IpmiToolPath = new ConfigKey<String>("Advanced", String.class, "outofbandmanagement.ipmitool.path", "/usr/bin/ipmitool",
+            "The out of band management ipmitool path used by the IpmiTool driver. Default: /usr/bin/ipmitool.", true, ConfigKey.Scope.Global);
+
+    public final ConfigKey<String> IpmiToolInterface = new ConfigKey<String>("Advanced", String.class, "outofbandmanagement.ipmitool.interface", "lanplus",
+            "The out of band management IpmiTool driver interface to use. Default: lanplus. Valid values are: lan, lanplus, open etc.", true, ConfigKey.Scope.Global);
+
+    public final ConfigKey<String> IpmiToolRetries = new ConfigKey<String>("Advanced", String.class, "outofbandmanagement.ipmitool.retries", "1",
+            "The out of band management IpmiTool driver retries option -R. Default 1.", true, ConfigKey.Scope.Global);
+
+    private String getIpmiUserId(ImmutableMap<OutOfBandManagement.Option, String> options, final Duration timeOut) {
+        final String username = options.get(OutOfBandManagement.Option.USERNAME);
+        if (Strings.isNullOrEmpty(username)) {
+            throw new CloudRuntimeException("Empty IPMI user configured, cannot proceed to find user's ID");
+        }
+
+        final List<String> ipmiToolCommands = IPMITOOL.getIpmiToolCommandArgs(IpmiToolPath.value(),
+                IpmiToolInterface.value(),
+                IpmiToolRetries.value(),
+                options, "user", "list");
+        final OutOfBandManagementDriverResponse output = IPMITOOL.executeCommands(ipmiToolCommands, timeOut);
+        if (!output.isSuccess()) {
+            throw new CloudRuntimeException("Failed to find IPMI user to change password, error: " + output.getError());
+        }
+
+        final String userId = IPMITOOL.findIpmiUser(output.getResult(), username);
+        if (Strings.isNullOrEmpty(userId)) {
+            throw new CloudRuntimeException("No IPMI user ID found for the username: " + username);
+        }
+        return userId;
+    }
+
+    public OutOfBandManagementDriverResponse execute(final OutOfBandManagementDriverCommand cmd) {
+        if (!isIpmiToolBinAvailable) {
+            initDriver();
+            if (!isIpmiToolBinAvailable) {
+                return new OutOfBandManagementDriverResponse(null, "Aborting operation due to ipmitool binary not available for execution", false);
+            }
+        }
+
+        OutOfBandManagementDriverResponse response = new OutOfBandManagementDriverResponse(null, "Unsupported Command", false);
+        if (!isDriverEnabled) {
+            response.setError("Driver not enabled or shutdown");
+            return response;
+        }
+        if (cmd instanceof OutOfBandManagementDriverPowerCommand) {
+            response = execute((OutOfBandManagementDriverPowerCommand) cmd);
+        } else if (cmd instanceof OutOfBandManagementDriverChangePasswordCommand) {
+            response = execute((OutOfBandManagementDriverChangePasswordCommand) cmd);
+        }
+
+        if (response != null && !response.isSuccess() && response.getError().contains("RAKP 2 HMAC is invalid")) {
+            response.setAuthFailure(true);
+        }
+        return response;
+    }
+
+    private OutOfBandManagementDriverResponse execute(final OutOfBandManagementDriverPowerCommand cmd) {
+        List<String> ipmiToolCommands = IPMITOOL.getIpmiToolCommandArgs(IpmiToolPath.value(),
+                IpmiToolInterface.value(),
+                IpmiToolRetries.value(),
+                cmd.getOptions(), "chassis", "power", IPMITOOL.parsePowerCommand(cmd.getPowerOperation()));
+
+        final OutOfBandManagementDriverResponse response = IPMITOOL.executeCommands(ipmiToolCommands, cmd.getTimeout());
+
+        if (response.isSuccess() && cmd.getPowerOperation().equals(OutOfBandManagement.PowerOperation.STATUS)) {
+            response.setPowerState(IPMITOOL.parsePowerState(response.getResult().trim()));
+        }
+        return response;
+    }
+
+    private OutOfBandManagementDriverResponse execute(final OutOfBandManagementDriverChangePasswordCommand cmd) {
+        final String outOfBandManagementUserId = getIpmiUserId(cmd.getOptions(), cmd.getTimeout());
+
+        final List<String> ipmiToolCommands = IPMITOOL.getIpmiToolCommandArgs(IpmiToolPath.value(),
+                IpmiToolInterface.value(), IpmiToolRetries.value(),
+                cmd.getOptions(), "user", "set", "password", outOfBandManagementUserId, cmd.getNewPassword());
+        return IPMITOOL.executeCommands(ipmiToolCommands, cmd.getTimeout());
+    }
+
+    private void initDriver() {
+        isDriverEnabled = true;
+        final OutOfBandManagementDriverResponse output = IPMITOOL.executeCommands(Arrays.asList(IpmiToolPath.value(), "-V"));
+        if (output.isSuccess() && output.getResult().startsWith("ipmitool version")) {
+            isIpmiToolBinAvailable = true;
+            LOG.debug("OutOfBandManagementDriver ipmitool initialized: " + output.getResult());
+        } else {
+            isIpmiToolBinAvailable = false;
+            LOG.error("OutOfBandManagementDriver ipmitool failed initialization with error: " + output.getError() + "; standard output: " + output.getResult());
+        }
+    }
+
+    private void stopDriver() {
+        isDriverEnabled = false;
+    }
+
+    @Override
+    public boolean start() {
+        initDriver();
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        ipmitoolExecutor.shutdown();
+        stopDriver();
+        return true;
+    }
+
+    @Override
+    public String getConfigComponentName() {
+        return IpmitoolOutOfBandManagementDriver.class.getSimpleName();
+    }
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys() {
+        return new ConfigKey<?>[] {IpmiToolPath, IpmiToolInterface, IpmiToolRetries};
+    }
+}

plugins/outofbandmanagement-drivers/ipmitool/src/org/apache/cloudstack/outofbandmanagement/driver/ipmitool/IpmitoolWrapper.java

@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement.driver.ipmitool;
+
+import com.cloud.utils.StringUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.cloudstack.utils.process.ProcessResult;
+import org.apache.cloudstack.utils.process.ProcessRunner;
+import org.apache.log4j.Logger;
+import org.joda.time.Duration;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+public final class IpmitoolWrapper {
+    public static final Logger LOG = Logger.getLogger(IpmitoolWrapper.class);
+
+    private final ProcessRunner RUNNER;
+
+    public IpmitoolWrapper(ExecutorService executor) {
+        this.RUNNER = new ProcessRunner(executor);
+    }
+
+    public String parsePowerCommand(OutOfBandManagement.PowerOperation operation) {
+        if (operation == null) {
+            throw new IllegalStateException("Invalid power operation requested");
+        }
+        switch (operation) {
+            case ON:
+            case OFF:
+            case CYCLE:
+            case RESET:
+            case SOFT:
+            case STATUS:
+                break;
+            default:
+                throw new IllegalStateException("Invalid power operation requested");
+        }
+        return operation.toString().toLowerCase();
+    }
+
+    public OutOfBandManagement.PowerState parsePowerState(final String standardOutput) {
+        if (Strings.isNullOrEmpty(standardOutput)) {
+            return OutOfBandManagement.PowerState.Unknown;
+        }
+        if (standardOutput.equals("Chassis Power is on")) {
+            return OutOfBandManagement.PowerState.On;
+        } else if (standardOutput.equals("Chassis Power is off")) {
+            return OutOfBandManagement.PowerState.Off;
+        }
+        return OutOfBandManagement.PowerState.Unknown;
+    }
+
+    public List<String> getIpmiToolCommandArgs(final String ipmiToolPath, final String ipmiInterface, final String retries,
+                                                      final ImmutableMap<OutOfBandManagement.Option, String> options, String... commands) {
+
+        final ImmutableList.Builder<String> ipmiToolCommands = ImmutableList.<String>builder()
+                                                            .add(ipmiToolPath)
+                                                            .add("-I")
+                                                            .add(ipmiInterface)
+                                                            .add("-R")
+                                                            .add(retries)
+                                                            .add("-v");
+
+        if (options != null) {
+            for (ImmutableMap.Entry<OutOfBandManagement.Option, String> option : options.entrySet()) {
+                switch (option.getKey()) {
+                    case ADDRESS:
+                        ipmiToolCommands.add("-H");
+                        break;
+                    case PORT:
+                        ipmiToolCommands.add("-p");
+                        break;
+                    case USERNAME:
+                        ipmiToolCommands.add("-U");
+                        break;
+                    case PASSWORD:
+                        ipmiToolCommands.add("-P");
+                        break;
+                    default:
+                        continue;
+                }
+                ipmiToolCommands.add(option.getValue());
+            }
+        }
+        for (String command : commands) {
+            ipmiToolCommands.add(command);
+        }
+        return ipmiToolCommands.build();
+    }
+
+    public String findIpmiUser(final String usersList, final String username) {
+        /**
+         * Expected usersList string contains legends on first line and users on rest
+         * ID Name  Callin Link Auth IPMI Msg Channel Priv Limit
+         * 1  admin true   true true ADMINISTRATOR
+         */
+
+        // Assuming user 'ID' index on 1st position
+        int idIndex = 0;
+
+        // Assuming  user 'Name' index on 2nd position
+        int usernameIndex = 1;
+
+        final String[] lines = usersList.split("\\r?\\n");
+        if (lines.length < 2) {
+            throw new CloudRuntimeException("Error parsing user ID from ipmi user listing");
+        }
+        // Find user and name indexes from the 1st line if not on default position
+        final String[] legends = lines[0].split(" +");
+        for (int idx = 0; idx < legends.length; idx++) {
+            if (legends[idx].equals("ID")) {
+                idIndex = idx;
+            }
+            if (legends[idx].equals("Name")) {
+                usernameIndex = idx;
+            }
+        }
+        // Find user 'ID' based on provided username and ID/Name positions
+        String userId = null;
+        for (int idx = 1; idx < lines.length; idx++) {
+            final String[] words = lines[idx].split(" +");
+            if (usernameIndex < words.length && idIndex < words.length) {
+                if (words[usernameIndex].equals(username)) {
+                    userId = words[idIndex];
+                }
+            }
+        }
+        return userId;
+    }
+
+    public OutOfBandManagementDriverResponse executeCommands(final List<String> commands) {
+        return executeCommands(commands, ProcessRunner.DEFAULT_MAX_TIMEOUT);
+    }
+
+    public OutOfBandManagementDriverResponse executeCommands(final List<String> commands, final Duration timeOut) {
+        final ProcessResult result = RUNNER.executeCommands(commands, timeOut);
+        if (LOG.isTraceEnabled()) {
+            List<String> cleanedCommands = new ArrayList<String>();
+            int maskNextCommand = 0;
+            for (String command : commands) {
+                if (maskNextCommand > 0) {
+                    cleanedCommands.add("**** ");
+                    maskNextCommand--;
+                    continue;
+                }
+                if (command.equalsIgnoreCase("-P")) {
+                    maskNextCommand = 1;
+                } else if (command.toLowerCase().endsWith("password")) {
+                    maskNextCommand = 2;
+                }
+                cleanedCommands.add(command);
+            }
+            LOG.trace("Executed ipmitool process with commands: " + StringUtils.join(cleanedCommands, ", ") +
+                      "\nIpmitool execution standard output: " + result.getStdOutput() +
+                      "\nIpmitool execution error output: " + result.getStdError());
+        }
+        return new OutOfBandManagementDriverResponse(result.getStdOutput(), result.getStdError(), result.isSuccess());
+    }
+}

plugins/outofbandmanagement-drivers/ipmitool/test/org/apache/cloudstack/outofbandmanagement/driver/ipmitool/IpmitoolWrapperTest.java

@@ -0,0 +1,115 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.outofbandmanagement.driver.ipmitool;
+
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IpmitoolWrapperTest {
+
+    private static final ExecutorService ipmitoolExecutor = Executors.newFixedThreadPool(20, new NamedThreadFactory("IpmiToolDriverTest"));
+    private static final IpmitoolWrapper IPMITOOL = new IpmitoolWrapper(ipmitoolExecutor);
+
+    @Test
+    public void testParsePowerCommandValid() {
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.ON), "on");
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.OFF), "off");
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.CYCLE), "cycle");
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.RESET), "reset");
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.SOFT), "soft");
+        Assert.assertEquals(IPMITOOL.parsePowerCommand(OutOfBandManagement.PowerOperation.STATUS), "status");
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testParsePowerCommandInvalid() {
+        IPMITOOL.parsePowerCommand(null);
+        Assert.fail("IpmitoolWrapper.parsePowerCommand failed to throw exception on invalid power state");
+    }
+
+    @Test
+    public void testParsePowerState() {
+        Assert.assertEquals(IPMITOOL.parsePowerState(null), OutOfBandManagement.PowerState.Unknown);
+        Assert.assertEquals(IPMITOOL.parsePowerState(""), OutOfBandManagement.PowerState.Unknown);
+        Assert.assertEquals(IPMITOOL.parsePowerState(" "), OutOfBandManagement.PowerState.Unknown);
+        Assert.assertEquals(IPMITOOL.parsePowerState("invalid data"), OutOfBandManagement.PowerState.Unknown);
+        Assert.assertEquals(IPMITOOL.parsePowerState("Chassis Power is on"), OutOfBandManagement.PowerState.On);
+        Assert.assertEquals(IPMITOOL.parsePowerState("Chassis Power is off"), OutOfBandManagement.PowerState.Off);
+    }
+
+    @Test
+    public void testGetIpmiToolCommandArgs() {
+        List<String> args = IPMITOOL.getIpmiToolCommandArgs("binpath", "intf", "1", null);
+        assert args != null;
+        Assert.assertEquals(args.size(), 6);
+        Assert.assertArrayEquals(args.toArray(), new String[]{"binpath", "-I", "intf", "-R", "1", "-v"});
+
+        ImmutableMap.Builder<OutOfBandManagement.Option, String> argMap = new ImmutableMap.Builder<>();
+        argMap.put(OutOfBandManagement.Option.DRIVER, "ipmitool");
+        argMap.put(OutOfBandManagement.Option.ADDRESS, "127.0.0.1");
+        List<String> argsWithOpts = IPMITOOL.getIpmiToolCommandArgs("binpath", "intf", "1", argMap.build(), "user", "list");
+        assert argsWithOpts != null;
+        Assert.assertEquals(argsWithOpts.size(), 10);
+        Assert.assertArrayEquals(argsWithOpts.toArray(), new String[]{"binpath", "-I", "intf", "-R", "1", "-v", "-H", "127.0.0.1", "user", "list"});
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testFindIpmiUserInvalid() {
+        IPMITOOL.findIpmiUser("some invalid string\n", "admin");
+        Assert.fail("IpmitoolWrapper.findIpmiUser failed to throw exception on invalid data");
+
+        Assert.assertEquals(IPMITOOL.findIpmiUser("some\ninvalid\ndata\n", "admin"), null);
+    }
+
+    @Test
+    public void testFindIpmiUserNull() {
+        Assert.assertEquals(IPMITOOL.findIpmiUser("some\ninvalid\ndata\n", "admin"), null);
+    }
+
+    @Test
+    public void testFindIpmiUserValid() {
+        String usersList = "ID  Name\t     Callin  Link Auth\tIPMI Msg   Channel Priv Limit\n" +
+                           "1   admin            true    true       true       ADMINISTRATOR\n" +
+                           "2   operator         true    false      false      OPERATOR\n" +
+                           "3   user             true    true       true       USER\n";
+        Assert.assertEquals(IPMITOOL.findIpmiUser(usersList, "admin"), "1");
+        Assert.assertEquals(IPMITOOL.findIpmiUser(usersList, "operator"), "2");
+        Assert.assertEquals(IPMITOOL.findIpmiUser(usersList, "user"), "3");
+    }
+
+    @Test
+    public void testExecuteCommands() {
+        OutOfBandManagementDriverResponse r = IPMITOOL.executeCommands(Arrays.asList("ls", "/tmp"));
+        Assert.assertTrue(r.isSuccess());
+        Assert.assertTrue(r.getResult().length() > 0);
+    }
+}

plugins/pom.xml

@@ -80,6 +80,7 @@
     <module>network-elements/midonet</module>
     <module>network-elements/stratosphere-ssp</module>
     <module>network-elements/opendaylight</module>
+    <module>outofbandmanagement-drivers/ipmitool</module>
     <module>storage-allocators/random</module>
     <module>user-authenticators/ldap</module>
     <module>user-authenticators/md5</module>

server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml

@@ -142,6 +142,10 @@
 
     <bean id="oCFS2ManagerImpl" class="com.cloud.storage.OCFS2ManagerImpl" />
 
+    <bean id="outOfBandManagementServiceImpl" class="org.apache.cloudstack.outofbandmanagement.OutOfBandManagementServiceImpl">
+        <property name="outOfBandManagementDrivers" value="#{outOfBandManagementDriversRegistry.registered}" />
+    </bean>
+
     <bean id="projectManagerImpl" class="com.cloud.projects.ProjectManagerImpl" />
 
     <bean id="queryManagerImpl" class="com.cloud.api.query.QueryManagerImpl" />

server/src/com/cloud/alert/AlertManagerImpl.java

@@ -758,7 +758,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
                 (alertType != AlertManager.AlertType.ALERT_TYPE_STORAGE_MISC) &&
                 (alertType != AlertManager.AlertType.ALERT_TYPE_MANAGMENT_NODE) &&
                 (alertType != AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED) &&
-                (alertType != AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED)) {
+                (alertType != AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED) &&
+                (alertType != AlertManager.AlertType.ALERT_TYPE_OOBM_AUTH_ERROR)) {
                 alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId);
             }
 

server/src/com/cloud/api/ApiResponseHelper.java

@@ -191,6 +191,7 @@ import com.cloud.configuration.Resource.ResourceOwnerType;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.configuration.ResourceCount;
 import com.cloud.configuration.ResourceLimit;
+import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.HostPodVO;
@@ -342,6 +343,8 @@ public class ApiResponseHelper implements ResponseGenerator {
     private SnapshotDataStoreDao _snapshotStoreDao;
     @Inject
     private PrimaryDataStoreDao _storagePoolDao;
+    @Inject
+    private ClusterDetailsDao _clusterDetailsDao;
 
     @Override
     public UserResponse createUserResponse(User user) {
@@ -1053,6 +1056,7 @@ public class ApiResponseHelper implements ResponseGenerator {
         String memoryOvercommitRatio = ApiDBUtils.findClusterDetails(cluster.getId(), "memoryOvercommitRatio");
         clusterResponse.setCpuOvercommitRatio(cpuOvercommitRatio);
         clusterResponse.setMemoryOvercommitRatio(memoryOvercommitRatio);
+        clusterResponse.setResourceDetails(_clusterDetailsDao.findDetails(cluster.getId()));
 
         if (showCapacities != null && showCapacities) {
             List<SummedCapacity> capacities = ApiDBUtils.getCapacityByClusterPodZone(null, null, cluster.getId());

server/src/com/cloud/api/query/QueryManagerImpl.java

@@ -1577,6 +1577,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         Object cluster = cmd.getClusterId();
         Object id = cmd.getId();
         Object keyword = cmd.getKeyword();
+        Object outOfBandManagementEnabled = cmd.isOutOfBandManagementEnabled();
+        Object powerState = cmd.getHostOutOfBandManagementPowerState();
         Object resourceState = cmd.getResourceState();
         Object haHosts = cmd.getHaHost();
         Long startIndex = cmd.getStartIndex();
@@ -1595,6 +1597,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         sb.and("dataCenterId", sb.entity().getZoneId(), SearchCriteria.Op.EQ);
         sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
         sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and("oobmEnabled", sb.entity().isOutOfBandManagementEnabled(), SearchCriteria.Op.EQ);
+        sb.and("powerState", sb.entity().getOutOfBandManagementPowerState(), SearchCriteria.Op.EQ);
         sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.EQ);
         sb.and("hypervisor_type", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ);
 
@@ -1644,6 +1648,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             sc.setParameters("clusterId", cluster);
         }
 
+        if (outOfBandManagementEnabled != null) {
+            sc.setParameters("oobmEnabled", outOfBandManagementEnabled);
+        }
+
+        if (powerState != null) {
+            sc.setParameters("powerState", powerState);
+        }
+
         if (resourceState != null) {
             sc.setParameters("resourceState", resourceState);
         }

server/src/com/cloud/api/query/dao/HostJoinDaoImpl.java

@@ -22,7 +22,6 @@ import java.util.Date;
 import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import javax.inject.Inject;
@@ -36,6 +35,7 @@ import org.apache.cloudstack.api.response.HostForMigrationResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.VgpuResponse;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.vo.HostJoinVO;
@@ -43,8 +43,7 @@ import com.cloud.gpu.HostGpuGroupsVO;
 import com.cloud.gpu.VGPUTypesVO;
 import com.cloud.host.Host;
 import com.cloud.host.HostStats;
-import com.cloud.host.HostVO;
-import com.cloud.host.dao.HostDao;
+import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.StorageStats;
 import com.cloud.utils.db.GenericDaoBase;
@@ -58,7 +57,9 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
     @Inject
     private ConfigurationDao _configDao;
     @Inject
-    private HostDao hostDao;
+    private HostDetailsDao hostDetailsDao;
+    @Inject
+    private OutOfBandManagementDao outOfBandManagementDao;
 
     private final SearchBuilder<HostJoinVO> hostSearch;
 
@@ -189,11 +190,7 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
             if (details.contains(HostDetails.all) && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
                 //only kvm has the requirement to return host details
                 try {
-                    HostVO h = hostDao.findById(host.getId());
-                    hostDao.loadDetails(h);
-                    Map<String, String> hostVoDetails;
-                    hostVoDetails = h.getDetails();
-                    hostResponse.setDetails(hostVoDetails);
+                    hostResponse.setDetails(hostDetailsDao.findDetails(host.getId()));
                 } catch (Exception e) {
                     s_logger.debug("failed to get host details", e);
                 }
@@ -225,6 +222,7 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
             }
         }
 
+        hostResponse.setOutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
         hostResponse.setResourceState(host.getResourceState().toString());
 
         // set async job

server/src/com/cloud/api/query/vo/HostJoinVO.java

@@ -36,6 +36,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.org.Cluster;
 import com.cloud.resource.ResourceState;
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
 
 /**
  * Host DB view.
@@ -91,6 +92,13 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity
     @Column(name = GenericDao.REMOVED_COLUMN)
     private Date removed;
 
+    @Column(name = "oobm_enabled")
+    private boolean outOfBandManagementEnabled = false;
+
+    @Column(name = "oobm_power_state")
+    @Enumerated(value = EnumType.STRING)
+    private OutOfBandManagement.PowerState outOfBandManagementPowerState;
+
     @Column(name = "resource_state")
     @Enumerated(value = EnumType.STRING)
     private ResourceState resourceState;
@@ -244,6 +252,14 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity
         return removed;
     }
 
+    public boolean isOutOfBandManagementEnabled() {
+        return outOfBandManagementEnabled;
+    }
+
+    public OutOfBandManagement.PowerState getOutOfBandManagementPowerState() {
+        return outOfBandManagementPowerState;
+    }
+
     public ResourceState getResourceState() {
         return resourceState;
     }

server/src/com/cloud/resource/ResourceManagerImpl.java

@@ -76,6 +76,7 @@ import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
 import com.cloud.dc.ClusterVO;
+import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenter.NetworkType;
 import com.cloud.dc.DataCenterIpAddressVO;
 import com.cloud.dc.DataCenterVO;
@@ -1349,6 +1350,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         return _clusterDao.findById(clusterId);
     }
 
+    @Override
+    public DataCenter getZone(Long zoneId) {
+        return _dcDao.findById(zoneId);
+    }
+
     @Override
     public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
         _defaultSystemVMHypervisor = HypervisorType.getType(_configDao.getValue(Config.SystemVMDefaultHypervisor.toString()));

server/src/com/cloud/server/ManagementServerImpl.java

@@ -133,6 +133,15 @@ import org.apache.cloudstack.api.command.admin.offering.DeleteDiskOfferingCmd;
 import org.apache.cloudstack.api.command.admin.offering.DeleteServiceOfferingCmd;
 import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd;
 import org.apache.cloudstack.api.command.admin.offering.UpdateServiceOfferingCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.DisableOutOfBandManagementForClusterCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.DisableOutOfBandManagementForHostCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.DisableOutOfBandManagementForZoneCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.EnableOutOfBandManagementForClusterCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.EnableOutOfBandManagementForHostCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.EnableOutOfBandManagementForZoneCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.ConfigureOutOfBandManagementCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.IssueOutOfBandManagementPowerActionCmd;
+import org.apache.cloudstack.api.command.admin.outofbandmanagement.ChangeOutOfBandManagementPasswordCmd;
 import org.apache.cloudstack.api.command.admin.pod.CreatePodCmd;
 import org.apache.cloudstack.api.command.admin.pod.DeletePodCmd;
 import org.apache.cloudstack.api.command.admin.pod.ListPodsByCmd;
@@ -3021,6 +3030,17 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(UpdateLBHealthCheckPolicyCmd.class);
         cmdList.add(GetUploadParamsForTemplateCmd.class);
         cmdList.add(GetUploadParamsForVolumeCmd.class);
+        // Out-of-band management APIs for admins
+        cmdList.add(EnableOutOfBandManagementForHostCmd.class);
+        cmdList.add(DisableOutOfBandManagementForHostCmd.class);
+        cmdList.add(EnableOutOfBandManagementForClusterCmd.class);
+        cmdList.add(DisableOutOfBandManagementForClusterCmd.class);
+        cmdList.add(EnableOutOfBandManagementForZoneCmd.class);
+        cmdList.add(DisableOutOfBandManagementForZoneCmd.class);
+        cmdList.add(ConfigureOutOfBandManagementCmd.class);
+        cmdList.add(IssueOutOfBandManagementPowerActionCmd.class);
+        cmdList.add(ChangeOutOfBandManagementPasswordCmd.class);
+
         return cmdList;
     }
 

server/src/com/cloud/server/StatsCollector.java

@@ -20,6 +20,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -33,6 +34,11 @@ import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementVO;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.usage.UsageUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -151,6 +157,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     @Inject
     private HostDao _hostDao;
     @Inject
+    private OutOfBandManagementDao outOfBandManagementDao;
+    @Inject
     private UserVmDao _userVmDao;
     @Inject
     private VolumeDao _volsDao;
@@ -167,6 +175,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     @Inject
     private ResourceManager _resourceMgr;
     @Inject
+    private OutOfBandManagementService outOfBandManagementService;
+    @Inject
     private ConfigurationDao _configDao;
     @Inject
     private EndPointSelector _epSelector;
@@ -208,6 +218,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>();
 
     long hostStatsInterval = -1L;
+    long hostOutOfBandManagementStatsInterval = -1L;
     long hostAndVmStatsInterval = -1L;
     long storageStatsInterval = -1L;
     long volumeStatsInterval = -1L;
@@ -251,8 +262,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     }
 
     private void init(Map<String, String> configs) {
-        _executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("StatsCollector"));
+        _executor = Executors.newScheduledThreadPool(6, new NamedThreadFactory("StatsCollector"));
 
+        hostOutOfBandManagementStatsInterval = OutOfBandManagementService.SyncThreadInterval.value();
         hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
         hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
         storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
@@ -294,6 +306,10 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             _executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS);
         }
 
+        if (hostOutOfBandManagementStatsInterval > 0) {
+            _executor.scheduleWithFixedDelay(new HostOutOfBandManagementStatsCollector(), 15000L, hostOutOfBandManagementStatsInterval, TimeUnit.MILLISECONDS);
+        }
+
         if (hostAndVmStatsInterval > 0) {
             _executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS);
         }
@@ -412,6 +428,36 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
         }
     }
 
+    class HostOutOfBandManagementStatsCollector extends ManagedContextRunnable {
+        @Override
+        protected void runInContext() {
+            try {
+                s_logger.debug("HostOutOfBandManagementStatsCollector is running...");
+                List<OutOfBandManagementVO> outOfBandManagementHosts = outOfBandManagementDao.findAllByManagementServer(ManagementServerNode.getManagementServerId());
+                if (outOfBandManagementHosts == null) {
+                    return;
+                }
+                for (OutOfBandManagement outOfBandManagementHost : outOfBandManagementHosts) {
+                    Host host = _hostDao.findById(outOfBandManagementHost.getHostId());
+                    if (host == null) {
+                        continue;
+                    }
+                    if (outOfBandManagementService.isOutOfBandManagementEnabled(host)) {
+                        outOfBandManagementService.submitBackgroundPowerSyncTask(host);
+                    } else if (outOfBandManagementHost.getPowerState() != OutOfBandManagement.PowerState.Disabled) {
+                        if (outOfBandManagementService.transitionPowerStateToDisabled(Collections.singletonList(host))) {
+                            if (s_logger.isDebugEnabled()) {
+                                s_logger.debug("Out-of-band management was disabled in zone/cluster/host, disabled power state for host id:" + host.getId());
+                            }
+                        }
+                    }
+                }
+            } catch (Throwable t) {
+                s_logger.error("Error trying to retrieve host out-of-band management stats", t);
+            }
+        }
+    }
+
     class VmStatsCollector extends ManagedContextRunnable {
         @Override
         protected void runInContext() {

server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementBackgroundTask.java

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.host.Host;
+import org.apache.log4j.Logger;
+
+public class OutOfBandManagementBackgroundTask implements Runnable {
+    public static final Logger LOG = Logger.getLogger(OutOfBandManagementBackgroundTask.class);
+
+    final private OutOfBandManagementService service;
+    final private Host host;
+    final private OutOfBandManagement.PowerOperation powerOperation;
+
+    public OutOfBandManagementBackgroundTask(OutOfBandManagementService service, Host host, OutOfBandManagement.PowerOperation powerOperation) {
+        this.service = service;
+        this.host = host;
+        this.powerOperation = powerOperation;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("[OOBM Task] Power operation:%s on Host:%d(%s)", powerOperation, host.getId(), host.getName());
+    }
+
+    @Override
+    public void run() {
+        try {
+            service.executeOutOfBandManagementPowerOperation(host, powerOperation, null);
+        } catch (Exception e) {
+            LOG.warn(String.format("Out-of-band management background task operation=%s for host id=%d failed with: %s",
+                    powerOperation.name(), host.getId(), e.getMessage()));
+        }
+    }
+}

server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java

@@ -0,0 +1,555 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.alert.AlertManager;
+import com.cloud.dc.ClusterDetailsDao;
+import com.cloud.dc.ClusterDetailsVO;
+import com.cloud.dc.DataCenter;
+import com.cloud.dc.DataCenterDetailVO;
+import com.cloud.dc.dao.DataCenterDetailsDao;
+import com.cloud.domain.Domain;
+import com.cloud.event.ActionEvent;
+import com.cloud.event.ActionEventUtils;
+import com.cloud.event.EventTypes;
+import com.cloud.host.Host;
+import com.cloud.host.dao.HostDao;
+import com.cloud.org.Cluster;
+import com.cloud.utils.component.Manager;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.db.GlobalLock;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallback;
+import com.cloud.utils.db.TransactionStatus;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+@Component
+public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOfBandManagementService, Manager, Configurable {
+    public static final Logger LOG = Logger.getLogger(OutOfBandManagementServiceImpl.class);
+
+    @Inject
+    private ClusterDetailsDao clusterDetailsDao;
+    @Inject
+    private DataCenterDetailsDao dataCenterDetailsDao;
+    @Inject
+    private OutOfBandManagementDao outOfBandManagementDao;
+    @Inject
+    private HostDao hostDao;
+    @Inject
+    private AlertManager alertMgr;
+
+    private String name;
+    private long serviceId;
+
+    private List<OutOfBandManagementDriver> outOfBandManagementDrivers = new ArrayList<>();
+    private final Map<String, OutOfBandManagementDriver> outOfBandManagementDriversMap = new HashMap<String, OutOfBandManagementDriver>();
+
+    private static final String OOBM_ENABLED_DETAIL = "outOfBandManagementEnabled";
+    private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST = 120;
+
+    private static Cache<Long, Long> hostAlertCache;
+    private static ExecutorService backgroundSyncBlockingExecutor;
+
+    private String getOutOfBandManagementHostLock(long id) {
+        return "oobm.host." + id;
+    }
+
+    private void initializeDriversMap() {
+        if (outOfBandManagementDriversMap.isEmpty() && outOfBandManagementDrivers != null && outOfBandManagementDrivers.size() > 0) {
+            for (final OutOfBandManagementDriver driver : outOfBandManagementDrivers) {
+                outOfBandManagementDriversMap.put(driver.getName().toLowerCase(), driver);
+            }
+            LOG.debug("Discovered out-of-band management drivers configured in the OutOfBandManagementService");
+        }
+    }
+
+    private OutOfBandManagementDriver getDriver(final OutOfBandManagement outOfBandManagementConfig) {
+        if (!Strings.isNullOrEmpty(outOfBandManagementConfig.getDriver())) {
+            final OutOfBandManagementDriver driver = outOfBandManagementDriversMap.get(outOfBandManagementConfig.getDriver());
+            if (driver != null) {
+                return driver;
+            }
+        }
+        throw new CloudRuntimeException("Configured out-of-band management driver is not available. Aborting any out-of-band management action.");
+    }
+
+    protected OutOfBandManagement updateConfig(final OutOfBandManagement outOfBandManagementConfig, final ImmutableMap<OutOfBandManagement.Option, String> options) {
+        if (outOfBandManagementConfig == null) {
+            throw new CloudRuntimeException("Out-of-band management is not configured for the host. Aborting.");
+        }
+        if (options == null) {
+            return outOfBandManagementConfig;
+        }
+        for (OutOfBandManagement.Option option: options.keySet()) {
+            final String value = options.get(option);
+            if (Strings.isNullOrEmpty(value)) {
+                continue;
+            }
+            switch (option) {
+                case DRIVER:
+                    outOfBandManagementConfig.setDriver(value);
+                    break;
+                case ADDRESS:
+                    outOfBandManagementConfig.setAddress(value);
+                    break;
+                case PORT:
+                    outOfBandManagementConfig.setPort(Integer.parseInt(value));
+                    break;
+                case USERNAME:
+                    outOfBandManagementConfig.setUsername(value);
+                    break;
+                case PASSWORD:
+                    outOfBandManagementConfig.setPassword(value);
+                    break;
+            }
+        }
+        return outOfBandManagementConfig;
+    }
+
+    protected ImmutableMap<OutOfBandManagement.Option, String> getOptions(final OutOfBandManagement outOfBandManagementConfig) {
+        final ImmutableMap.Builder<OutOfBandManagement.Option, String> optionsBuilder = ImmutableMap.builder();
+        if (outOfBandManagementConfig == null) {
+            throw new CloudRuntimeException("Out-of-band management is not configured for the host. Aborting.");
+        }
+        for (OutOfBandManagement.Option option: OutOfBandManagement.Option.values()) {
+            String value = null;
+            switch (option) {
+                case DRIVER:
+                    value = outOfBandManagementConfig.getDriver();
+                    break;
+                case ADDRESS:
+                    value = outOfBandManagementConfig.getAddress();
+                    break;
+                case PORT:
+                    if (outOfBandManagementConfig.getPort() != null) {
+                        value = String.valueOf(outOfBandManagementConfig.getPort());
+                    }
+                    break;
+                case USERNAME:
+                    value = outOfBandManagementConfig.getUsername();
+                    break;
+                case PASSWORD:
+                    value = outOfBandManagementConfig.getPassword();
+                    break;
+            }
+            if (value != null) {
+                optionsBuilder.put(option, value);
+            }
+        }
+        return optionsBuilder.build();
+    }
+
+    private void sendAuthError(final Host host, final String message) {
+        try {
+            hostAlertCache.asMap().putIfAbsent(host.getId(), 0L);
+            Long sentCount = hostAlertCache.asMap().get(host.getId());
+            if (sentCount != null && sentCount <= 0) {
+                boolean concurrentUpdateResult = hostAlertCache.asMap().replace(host.getId(), sentCount, sentCount+1L);
+                if (concurrentUpdateResult) {
+                    final String subject = String.format("Out-of-band management auth-error detected for host:%d in cluster:%d, zone:%d", host.getId(), host.getClusterId(), host.getDataCenterId());
+                    LOG.error(subject + ": " + message);
+                    alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_OOBM_AUTH_ERROR, host.getDataCenterId(), host.getPodId(), subject, message);
+                }
+            }
+        } catch (Exception ignored) {
+        }
+    }
+
+    private boolean transitionPowerState(OutOfBandManagement.PowerState.Event event, OutOfBandManagement outOfBandManagementHost) {
+        if (outOfBandManagementHost == null) {
+            return false;
+        }
+        OutOfBandManagement.PowerState currentPowerState = outOfBandManagementHost.getPowerState();
+        try {
+            OutOfBandManagement.PowerState newPowerState = OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState, event);
+            boolean result = outOfBandManagementDao.updateState(currentPowerState, event, newPowerState, outOfBandManagementHost, null);
+            if (result) {
+                final String message = String.format("Transitioned out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId());
+                LOG.debug(message);
+                ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
+                        EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
+            }
+            return result;
+        } catch (NoTransitionException ignored) {
+            LOG.trace(String.format("Unable to transition out-of-band management power state for host id=%s for the event=%s and current power state=%s", outOfBandManagementHost.getHostId(), event, currentPowerState));
+        }
+        return false;
+    }
+
+    private boolean isOutOfBandManagementEnabledForZone(Long zoneId) {
+        if (zoneId == null) {
+            return true;
+        }
+        final DataCenterDetailVO zoneDetails = dataCenterDetailsDao.findDetail(zoneId, OOBM_ENABLED_DETAIL);
+        if (zoneDetails != null && !Strings.isNullOrEmpty(zoneDetails.getValue()) && !Boolean.valueOf(zoneDetails.getValue())) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean isOutOfBandManagementEnabledForCluster(Long clusterId) {
+        if (clusterId == null) {
+            return true;
+        }
+        final ClusterDetailsVO clusterDetails = clusterDetailsDao.findDetail(clusterId, OOBM_ENABLED_DETAIL);
+        if (clusterDetails != null && !Strings.isNullOrEmpty(clusterDetails.getValue()) && !Boolean.valueOf(clusterDetails.getValue())) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean isOutOfBandManagementEnabledForHost(Long hostId) {
+        if (hostId == null) {
+            return false;
+        }
+        final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(hostId);
+        if (outOfBandManagementConfig == null || !outOfBandManagementConfig.isEnabled()) {
+            return false;
+        }
+        return true;
+    }
+
+    private void checkOutOfBandManagementEnabledByZoneClusterHost(final Host host) {
+        if (!isOutOfBandManagementEnabledForZone(host.getDataCenterId())) {
+            throw new CloudRuntimeException("Out-of-band management is disabled for the host's zone. Aborting Operation.");
+        }
+        if (!isOutOfBandManagementEnabledForCluster(host.getClusterId())) {
+            throw new CloudRuntimeException("Out-of-band management is disabled for the host's cluster. Aborting Operation.");
+        }
+        if (!isOutOfBandManagementEnabledForHost(host.getId())) {
+            throw new CloudRuntimeException("Out-of-band management is disabled or not configured for the host. Aborting Operation.");
+        }
+    }
+
+    public boolean isOutOfBandManagementEnabled(final Host host) {
+        return isOutOfBandManagementEnabledForZone(host.getDataCenterId())
+                && isOutOfBandManagementEnabledForCluster(host.getClusterId())
+                && isOutOfBandManagementEnabledForHost(host.getId());
+    }
+
+    public boolean transitionPowerStateToDisabled(List<? extends Host> hosts) {
+        boolean result = true;
+        for (Host host : hosts) {
+            result = result && transitionPowerState(OutOfBandManagement.PowerState.Event.Disabled,
+                    outOfBandManagementDao.findByHost(host.getId()));
+        }
+        return result;
+    }
+
+    public void submitBackgroundPowerSyncTask(final Host host) {
+        if (host != null) {
+            backgroundSyncBlockingExecutor.submit(new OutOfBandManagementBackgroundTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
+        }
+    }
+
+    private OutOfBandManagementResponse buildEnableDisableResponse(final boolean enabled) {
+        final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
+        response.setEnabled(enabled);
+        response.setSuccess(true);
+        return response;
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE, eventDescription = "enabling out-of-band management on a zone")
+    public OutOfBandManagementResponse enableOutOfBandManagement(final DataCenter zone) {
+        dataCenterDetailsDao.persist(zone.getId(), OOBM_ENABLED_DETAIL, String.valueOf(true));
+        return buildEnableDisableResponse(true);
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, eventDescription = "disabling out-of-band management on a zone")
+    public OutOfBandManagementResponse disableOutOfBandManagement(final DataCenter zone) {
+        dataCenterDetailsDao.persist(zone.getId(), OOBM_ENABLED_DETAIL, String.valueOf(false));
+        transitionPowerStateToDisabled(hostDao.findByDataCenterId(zone.getId()));
+
+        return buildEnableDisableResponse(false);
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE, eventDescription = "enabling out-of-band management on a cluster")
+    public OutOfBandManagementResponse enableOutOfBandManagement(final Cluster cluster) {
+        clusterDetailsDao.persist(cluster.getId(), OOBM_ENABLED_DETAIL, String.valueOf(true));
+        return buildEnableDisableResponse(true);
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, eventDescription = "disabling out-of-band management on a cluster")
+    public OutOfBandManagementResponse disableOutOfBandManagement(final Cluster cluster) {
+        clusterDetailsDao.persist(cluster.getId(), OOBM_ENABLED_DETAIL, String.valueOf(false));
+        transitionPowerStateToDisabled(hostDao.findByClusterId(cluster.getId()));
+        return buildEnableDisableResponse(false);
+    }
+
+    private OutOfBandManagement getConfigForHost(final Host host) {
+        final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
+        if (outOfBandManagementConfig == null) {
+            throw new CloudRuntimeException("Out-of-band management is not configured for the host. Please configure the host before enabling/disabling it.");
+        }
+        return outOfBandManagementConfig;
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLE, eventDescription = "enabling out-of-band management on a host")
+    public OutOfBandManagementResponse enableOutOfBandManagement(final Host host) {
+        final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host);
+        hostAlertCache.invalidate(host.getId());
+        outOfBandManagementConfig.setEnabled(true);
+        boolean updateResult = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
+        if (updateResult) {
+            transitionPowerStateToDisabled(Collections.singletonList(host));
+        }
+        return buildEnableDisableResponse(true);
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_DISABLE, eventDescription = "disabling out-of-band management on a host")
+    public OutOfBandManagementResponse disableOutOfBandManagement(final Host host) {
+        final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host);
+        hostAlertCache.invalidate(host.getId());
+        outOfBandManagementConfig.setEnabled(false);
+        boolean updateResult = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
+        if (updateResult) {
+            transitionPowerStateToDisabled(Collections.singletonList(host));
+        }
+        return buildEnableDisableResponse(false);
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription = "updating out-of-band management configuration")
+    public OutOfBandManagementResponse configureOutOfBandManagement(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
+        OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
+        if (outOfBandManagementConfig == null) {
+            outOfBandManagementConfig = outOfBandManagementDao.persist(new OutOfBandManagementVO(host.getId()));
+        }
+        outOfBandManagementConfig = updateConfig(outOfBandManagementConfig, options);
+        if (Strings.isNullOrEmpty(outOfBandManagementConfig.getDriver()) || !outOfBandManagementDriversMap.containsKey(outOfBandManagementConfig.getDriver().toLowerCase())) {
+            throw new CloudRuntimeException("Out-of-band management driver is not available. Please provide a valid driver name.");
+        }
+
+        boolean updatedConfig = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
+        CallContext.current().setEventDetails("host id:" + host.getId() + " configuration:" + outOfBandManagementConfig.getAddress() + ":" + outOfBandManagementConfig.getPort());
+
+        if (!updatedConfig) {
+            throw new CloudRuntimeException("Failed to update out-of-band management config for the host in the database.");
+        }
+
+        String result = "Out-of-band management successfully configured for the host";
+        LOG.debug(result);
+
+        final OutOfBandManagementResponse response = new OutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
+        response.setResultDescription(result);
+        response.setSuccess(true);
+        return response;
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing host out-of-band management action", async = true)
+    public OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
+        checkOutOfBandManagementEnabledByZoneClusterHost(host);
+        final OutOfBandManagement outOfBandManagementConfig = getConfigForHost(host);
+        final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
+        final OutOfBandManagementDriver driver = getDriver(outOfBandManagementConfig);
+
+        Long actionTimeOut = timeout;
+        if (actionTimeOut == null) {
+            actionTimeOut = ActionTimeout.valueIn(host.getClusterId());
+        }
+
+        final OutOfBandManagementDriverPowerCommand cmd = new OutOfBandManagementDriverPowerCommand(options, actionTimeOut, powerOperation);
+        final OutOfBandManagementDriverResponse driverResponse = driver.execute(cmd);
+
+        if (driverResponse == null) {
+            throw new CloudRuntimeException(String.format("Out-of-band Management action (%s) on host (%s) failed due to no response from the driver", powerOperation, host.getUuid()));
+        }
+
+        if (powerOperation.equals(OutOfBandManagement.PowerOperation.STATUS)) {
+            transitionPowerState(driverResponse.toEvent(), outOfBandManagementConfig);
+        }
+
+        if (!driverResponse.isSuccess()) {
+            String errorMessage = String.format("Out-of-band Management action (%s) on host (%s) failed with error: %s", powerOperation, host.getUuid(), driverResponse.getError());
+            if (driverResponse.hasAuthFailure()) {
+                errorMessage = String.format("Out-of-band Management action (%s) on host (%s) failed due to authentication error: %s. Please check configured credentials.", powerOperation, host.getUuid(), driverResponse.getError());
+                sendAuthError(host, errorMessage);
+            }
+            if (!powerOperation.equals(OutOfBandManagement.PowerOperation.STATUS)) {
+                LOG.debug(errorMessage);
+            }
+            throw new CloudRuntimeException(errorMessage);
+        }
+
+        final OutOfBandManagementResponse response = new OutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
+        response.setSuccess(driverResponse.isSuccess());
+        response.setResultDescription(driverResponse.getResult());
+        response.setId(host.getUuid());
+        response.setOutOfBandManagementAction(powerOperation.toString());
+        return response;
+    }
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription = "updating out-of-band management password")
+    public OutOfBandManagementResponse changeOutOfBandManagementPassword(final Host host, final String newPassword) {
+        checkOutOfBandManagementEnabledByZoneClusterHost(host);
+        if (Strings.isNullOrEmpty(newPassword)) {
+            throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as provided new-password is null or empty for the host %s.", host.getUuid()));
+        }
+        GlobalLock outOfBandManagementHostLock = GlobalLock.getInternLock(getOutOfBandManagementHostLock(host.getId()));
+        try {
+            if (outOfBandManagementHostLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST)) {
+                try {
+                    final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
+
+                    final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
+                    if (!(options.containsKey(OutOfBandManagement.Option.PASSWORD) && !Strings.isNullOrEmpty(options.get(OutOfBandManagement.Option.PASSWORD)))) {
+                        throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as we've no previously configured password for the host %s.", host.getUuid()));
+                    }
+                    final OutOfBandManagementDriver driver = getDriver(outOfBandManagementConfig);
+
+                    final OutOfBandManagementDriverChangePasswordCommand cmd = new OutOfBandManagementDriverChangePasswordCommand(options, ActionTimeout.valueIn(host.getClusterId()), newPassword);
+                    final OutOfBandManagementDriverResponse driverResponse;
+                    try {
+                        driverResponse = driver.execute(cmd);
+                    } catch (Exception e) {
+                        LOG.error("Out-of-band management change password failed due to driver error: " + e.getMessage());
+                        throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) due to driver error: %s", host.getUuid(), e.getMessage()));
+                    }
+
+                    if (!driverResponse.isSuccess()) {
+                        throw new CloudRuntimeException(String.format("Failed to change out-of-band management password for host (%s) with error: %s", host.getUuid(), driverResponse.getError()));
+                    }
+
+                    final boolean updatedConfigResult = Transaction.execute(new TransactionCallback<Boolean>() {
+                        @Override
+                        public Boolean doInTransaction(TransactionStatus status) {
+                            OutOfBandManagement updatedOutOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
+                            updatedOutOfBandManagementConfig.setPassword(newPassword);
+                            return outOfBandManagementDao.update(updatedOutOfBandManagementConfig.getId(), (OutOfBandManagementVO) updatedOutOfBandManagementConfig);
+                        }
+                    });
+
+                    if (!updatedConfigResult) {
+                        LOG.error(String.format("Succeeded to change out-of-band management password but failed to updated in database the new password:%s for the host id:%d", newPassword, host.getId()));
+                    }
+
+                    final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
+                    response.setSuccess(updatedConfigResult && driverResponse.isSuccess());
+                    response.setResultDescription(driverResponse.getResult());
+                    response.setId(host.getUuid());
+                    return response;
+                } finally {
+                    outOfBandManagementHostLock.unlock();
+                }
+            } else {
+                LOG.error("Unable to acquire synchronization lock to change out-of-band management password for host id: " + host.getId());
+                throw new CloudRuntimeException(String.format("Unable to acquire lock to change out-of-band management password for host (%s), please try after some time.", host.getUuid()));
+            }
+        } finally {
+            outOfBandManagementHostLock.releaseRef();
+        }
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public long getId() {
+        return serviceId;
+    }
+
+    @Override
+    public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
+        this.name = name;
+        this.serviceId = ManagementServerNode.getManagementServerId();
+
+        final int poolSize = SyncThreadPoolSize.value();
+
+        hostAlertCache = CacheBuilder.newBuilder()
+                .concurrencyLevel(4)
+                .weakKeys()
+                .maximumSize(100 * poolSize)
+                .expireAfterWrite(1, TimeUnit.DAYS)
+                .build();
+
+        backgroundSyncBlockingExecutor = new ThreadPoolExecutor(poolSize, poolSize,
+                0L, TimeUnit.MILLISECONDS,
+                new ArrayBlockingQueue<Runnable>(10 * poolSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
+
+        LOG.info("Starting out-of-band management background sync executor with thread pool-size=" + poolSize + " and background sync thread interval=" + SyncThreadInterval.value() + "s");
+        return true;
+    }
+
+    @Override
+    public boolean start() {
+        initializeDriversMap();
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        backgroundSyncBlockingExecutor.shutdown();
+        outOfBandManagementDao.expireOutOfBandManagementOwnershipByServer(getId());
+        return true;
+    }
+
+    @Override
+    public String getConfigComponentName() {
+        return OutOfBandManagementServiceImpl.class.getSimpleName();
+    }
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys() {
+        return new ConfigKey<?>[] {ActionTimeout, SyncThreadInterval, SyncThreadPoolSize};
+    }
+
+    public List<OutOfBandManagementDriver> getOutOfBandManagementDrivers() {
+        return outOfBandManagementDrivers;
+    }
+
+    public void setOutOfBandManagementDrivers(List<OutOfBandManagementDriver> outOfBandManagementDrivers) {
+        this.outOfBandManagementDrivers = outOfBandManagementDrivers;
+    }
+}

server/test/com/cloud/resource/MockResourceManagerImpl.java

@@ -37,6 +37,7 @@ import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupRoutingCommand;
 import com.cloud.agent.api.VgpuTypesInfo;
 import com.cloud.agent.api.to.GPUDeviceTO;
+import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.HostPodVO;
 import com.cloud.dc.PodCluster;
@@ -172,6 +173,12 @@ public class MockResourceManagerImpl extends ManagerBase implements ResourceMana
         return null;
     }
 
+    @Override
+    public DataCenter getZone(Long zoneId) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
     /* (non-Javadoc)
      * @see com.cloud.resource.ResourceService#getSupportedHypervisorTypes(long, boolean, java.lang.Long)
      */

server/test/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceTest.java

@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.outofbandmanagement;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class OutOfBandManagementServiceTest {
+
+    OutOfBandManagementServiceImpl oobmService = new OutOfBandManagementServiceImpl();
+
+    @Test
+    public void testOutOfBandManagementDriverResponseEvent() {
+        OutOfBandManagementDriverResponse r = new OutOfBandManagementDriverResponse("some result", "some error", false);
+
+        r.setSuccess(false);
+        r.setAuthFailure(false);
+        Assert.assertEquals(r.toEvent(), OutOfBandManagement.PowerState.Event.Unknown);
+
+        r.setSuccess(false);
+        r.setAuthFailure(true);
+        Assert.assertEquals(r.toEvent(), OutOfBandManagement.PowerState.Event.AuthError);
+
+        r.setAuthFailure(false);
+        r.setSuccess(true);
+        r.setPowerState(OutOfBandManagement.PowerState.On);
+        Assert.assertEquals(r.toEvent(), OutOfBandManagement.PowerState.Event.On);
+
+        r.setPowerState(OutOfBandManagement.PowerState.Off);
+        Assert.assertEquals(r.toEvent(), OutOfBandManagement.PowerState.Event.Off);
+
+        r.setPowerState(OutOfBandManagement.PowerState.Disabled);
+        Assert.assertEquals(r.toEvent(), OutOfBandManagement.PowerState.Event.Disabled);
+    }
+
+    private ImmutableMap<OutOfBandManagement.Option, String> buildRandomOptionsMap() {
+        ImmutableMap.Builder<OutOfBandManagement.Option, String> builder = new ImmutableMap.Builder<>();
+        builder.put(OutOfBandManagement.Option.ADDRESS, "localhost");
+        builder.put(OutOfBandManagement.Option.DRIVER, "ipmitool");
+        return builder.build();
+    }
+
+    @Test
+    public void testUpdateOutOfBandManagementConfigValid() {
+        OutOfBandManagement config = new OutOfBandManagementVO(123L);
+        Assert.assertEquals(config.getPowerState(), OutOfBandManagement.PowerState.Disabled);
+        config = oobmService.updateConfig(config, buildRandomOptionsMap());
+        Assert.assertEquals(config.getAddress(), "localhost");
+        Assert.assertEquals(config.getDriver(), "ipmitool");
+        Assert.assertEquals(config.getPowerState(), OutOfBandManagement.PowerState.Disabled);
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testUpdateOutOfBandManagementNullConfigValidOptions() {
+        oobmService.updateConfig(null, buildRandomOptionsMap());
+        Assert.fail("CloudRuntimeException was expect for out-of-band management not configured for the host");
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testUpdateOutOfBandManagementNullConfigNullOptions() {
+        oobmService.updateConfig(null, null);
+        Assert.fail("CloudRuntimeException was expect for out-of-band management not configured for the host");
+    }
+
+    @Test
+    public void testUpdateOutOfBandManagementValidConfigValidOptions() {
+        OutOfBandManagement config = new OutOfBandManagementVO(123L);
+        config.setAddress(null);
+        config = oobmService.updateConfig(config, null);
+        Assert.assertEquals(config.getAddress(), null);
+        Assert.assertEquals(config.getPowerState(), OutOfBandManagement.PowerState.Disabled);
+    }
+
+    @Test
+    public void testGetOutOfBandManagementOptionsValid() {
+        OutOfBandManagement configEmpty = new OutOfBandManagementVO(123L);
+        ImmutableMap<OutOfBandManagement.Option, String> optionsEmpty = oobmService.getOptions(configEmpty);
+        Assert.assertEquals(optionsEmpty.size(), 0);
+
+        OutOfBandManagement config = new OutOfBandManagementVO(123L);
+        config.setAddress("localhost");
+        config.setDriver("ipmitool");
+        config.setPort(1234);
+        ImmutableMap<OutOfBandManagement.Option, String> options = oobmService.getOptions(config);
+        Assert.assertEquals(options.get(OutOfBandManagement.Option.ADDRESS), "localhost");
+        Assert.assertEquals(options.get(OutOfBandManagement.Option.DRIVER), "ipmitool");
+        Assert.assertEquals(options.get(OutOfBandManagement.Option.PORT), "1234");
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testGetOutOfBandManagementOptionsInvalid() {
+        oobmService.getOptions(null);
+        Assert.fail("CloudRuntimeException was expected for finding options of host with out-of-band management configuration");
+    }
+}

setup/db/db/schema-481to490-cleanup.sql

@@ -271,3 +271,80 @@ CREATE VIEW `cloud`.`user_view` AS
         `cloud`.`async_job` ON async_job.instance_id = user.id
             and async_job.instance_type = 'User'
             and async_job.job_status = 0;
+
+-- Out-of-band management
+DROP VIEW IF EXISTS `cloud`.`host_view`;
+CREATE VIEW `cloud`.`host_view` AS
+    select
+        host.id,
+        host.uuid,
+        host.name,
+        host.status,
+        host.disconnected,
+        host.type,
+        host.private_ip_address,
+        host.version,
+        host.hypervisor_type,
+        host.hypervisor_version,
+        host.capabilities,
+        host.last_ping,
+        host.created,
+        host.removed,
+        host.resource_state,
+        host.mgmt_server_id,
+        host.cpu_sockets,
+        host.cpus,
+        host.speed,
+        host.ram,
+        cluster.id cluster_id,
+        cluster.uuid cluster_uuid,
+        cluster.name cluster_name,
+        cluster.cluster_type,
+        data_center.id data_center_id,
+        data_center.uuid data_center_uuid,
+        data_center.name data_center_name,
+        data_center.networktype data_center_type,
+        host_pod_ref.id pod_id,
+        host_pod_ref.uuid pod_uuid,
+        host_pod_ref.name pod_name,
+        host_tags.tag,
+        guest_os_category.id guest_os_category_id,
+        guest_os_category.uuid guest_os_category_uuid,
+        guest_os_category.name guest_os_category_name,
+        mem_caps.used_capacity memory_used_capacity,
+        mem_caps.reserved_capacity memory_reserved_capacity,
+        cpu_caps.used_capacity cpu_used_capacity,
+        cpu_caps.reserved_capacity cpu_reserved_capacity,
+        async_job.id job_id,
+        async_job.uuid job_uuid,
+        async_job.job_status job_status,
+        async_job.account_id job_account_id,
+        oobm.enabled AS `oobm_enabled`,
+        oobm.power_state AS `oobm_power_state`
+    from
+        `cloud`.`host`
+            left join
+        `cloud`.`cluster` ON host.cluster_id = cluster.id
+            left join
+        `cloud`.`data_center` ON host.data_center_id = data_center.id
+            left join
+        `cloud`.`host_pod_ref` ON host.pod_id = host_pod_ref.id
+            left join
+        `cloud`.`host_details` ON host.id = host_details.host_id
+            and host_details.name = 'guest.os.category.id'
+            left join
+        `cloud`.`guest_os_category` ON guest_os_category.id = CONVERT( host_details.value , UNSIGNED)
+            left join
+        `cloud`.`host_tags` ON host_tags.host_id = host.id
+            left join
+        `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id
+            and mem_caps.capacity_type = 0
+            left join
+        `cloud`.`op_host_capacity` cpu_caps ON host.id = cpu_caps.host_id
+            and cpu_caps.capacity_type = 1
+            left join
+        `cloud`.`async_job` ON async_job.instance_id = host.id
+            and async_job.instance_type = 'Host'
+            and async_job.job_status = 0
+            left join
+        `cloud`.`oobm` ON oobm.host_id = host.id;

setup/db/db/schema-481to490.sql

@@ -471,3 +471,26 @@ INSERT INTO `cloud`.`roles` (`id`, `uuid`, `name`, `role_type`, `description`) v
 INSERT INTO `cloud`.`roles` (`id`, `uuid`, `name`, `role_type`, `description`) values (2, UUID(), 'Resource Admin', 'ResourceAdmin', 'Default resource admin role') ON DUPLICATE KEY UPDATE name=name;
 INSERT INTO `cloud`.`roles` (`id`, `uuid`, `name`, `role_type`, `description`) values (3, UUID(), 'Domain Admin', 'DomainAdmin', 'Default domain admin role') ON DUPLICATE KEY UPDATE name=name;
 INSERT INTO `cloud`.`roles` (`id`, `uuid`, `name`, `role_type`, `description`) values (4, UUID(), 'User', 'User', 'Default Root Admin role') ON DUPLICATE KEY UPDATE name=name;
+
+-- Out-of-band management
+CREATE TABLE IF NOT EXISTS `cloud`.`oobm` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `host_id` bigint(20) unsigned DEFAULT NULL COMMENT 'foreign key to host',
+  `enabled` int(1) unsigned DEFAULT '0' COMMENT 'is out-of-band management enabled for host',
+  `power_state` varchar(32) DEFAULT 'Disabled' COMMENT 'out-of-band management power status',
+  `driver` varchar(32) DEFAULT NULL COMMENT 'out-of-band management driver',
+  `address` varchar(255) DEFAULT NULL COMMENT 'out-of-band management interface address',
+  `port` int(10) unsigned DEFAULT NULL COMMENT 'out-of-band management interface port',
+  `username` varchar(255) DEFAULT NULL COMMENT 'out-of-band management interface username',
+  `password` varchar(255) DEFAULT NULL COMMENT 'out-of-band management interface password',
+  `update_count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'atomic increase count making status update operation atomical',
+  `update_time` datetime COMMENT 'last power state update datetime',
+  `mgmt_server_id` bigint(20) unsigned DEFAULT NULL COMMENT 'management server id which owns out-of-band management for the host',
+  PRIMARY KEY (`id`),
+  KEY `fk_oobm__host_id` (`host_id`),
+  KEY `i_oobm__enabled` (`enabled`),
+  KEY `i_oobm__power_state` (`power_state`),
+  KEY `i_oobm__update_time` (`update_time`),
+  KEY `i_oobm__mgmt_server_id` (`mgmt_server_id`),
+  CONSTRAINT `fk_oobm__host_id` FOREIGN KEY (`host_id`) REFERENCES `host` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

setup/dev/advanced.cfg

@@ -154,7 +154,7 @@
         },
         {
             "name": "storage.cleanup.interval",
-            "value": "150"
+            "value": "60"
         },
         {
             "name": "vm.op.wait.interval",
@@ -162,7 +162,7 @@
         },
         {
             "name": "default.page.size",
-            "value": "10000"
+            "value": "500"
         },
         {
             "name": "network.gc.interval",
@@ -215,6 +215,18 @@
         {
             "name": "enable.dynamic.scale.vm",
             "value": "true"
+        },
+        {
+            "name": "ping.interval",
+            "value": "10"
+        },
+        {
+            "name": "ping.timeout",
+            "value": "1.5"
+        },
+        {
+            "name": "outofbandmanagement.sync.interval",
+            "value": "2000"
         }
     ],
     "mgtSvr": [

test/integration/smoke/test_outofbandmanagement.py

@@ -0,0 +1,575 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import marvin
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.lib.utils import *
+from marvin.lib.base import *
+from marvin.lib.common import *
+from marvin.lib.utils import (random_gen)
+from nose.plugins.attrib import attr
+
+from ipmisim.ipmisim import IpmiServerContext, IpmiServer, ThreadedIpmiServer
+
+import random
+import socket
+import sys
+import thread
+import time
+
+
+class TestOutOfBandManagement(cloudstackTestCase):
+    """ Test cases for out of band management
+    """
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.hypervisor = self.testClient.getHypervisorInfo()
+        self.dbclient = self.testClient.getDbConnection()
+        self.services = self.testClient.getParsedTestDataConfig()
+        self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__
+        self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20)
+
+        self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
+        self.host = None
+        self.server = None
+
+        # use random port for ipmisim
+        s = socket.socket()
+        s.bind(('', 0))
+        self.serverPort = s.getsockname()[1]
+        s.close()
+
+        self.cleanup = []
+
+
+    def tearDown(self):
+        try:
+            self.dbclient.execute("delete from oobm where port=%d" % self.getIpmiServerPort())
+            self.dbclient.execute("delete from mshost_peer where peer_runid=%s" % self.getFakeMsRunId())
+            self.dbclient.execute("delete from mshost where runid=%s" % self.getFakeMsRunId())
+            self.dbclient.execute("delete from cluster_details where name='outOfBandManagementEnabled'")
+            self.dbclient.execute("delete from data_center_details where name='outOfBandManagementEnabled'")
+            cleanup_resources(self.apiclient, self.cleanup)
+            if self.server:
+                self.server.shutdown()
+                self.server.server_close()
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+
+    def getFakeMsId(self):
+        return self.fakeMsId
+
+
+    def getFakeMsRunId(self):
+        return self.fakeMsId * 1000
+
+
+    def getHost(self, hostId=None):
+        if self.host and hostId is None:
+            return self.host
+
+        response = list_hosts(
+                        self.apiclient,
+                        zoneid=self.zone.id,
+                        type='Routing',
+                        id=hostId
+                        )
+        if len(response) > 0:
+            self.host = response[0]
+            return self.host
+        raise self.skipTest("No hosts found, skipping out-of-band management test")
+
+
+    def getIpmiServerIp(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect((self.mgtSvrDetails["mgtSvrIp"], self.mgtSvrDetails["port"]))
+        return s.getsockname()[0]
+
+
+    def getIpmiServerPort(self):
+        return self.serverPort
+
+
+    def getOobmConfigCmd(self):
+        cmd = configureOutOfBandManagement.configureOutOfBandManagementCmd()
+        cmd.driver = 'ipmitool' # The default available driver
+        cmd.address = self.getIpmiServerIp()
+        cmd.port = self.getIpmiServerPort()
+        cmd.username = 'admin'
+        cmd.password = 'password'
+        cmd.hostid = self.getHost().id
+        return cmd
+
+
+    def getOobmEnableCmd(self):
+        cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
+        cmd.hostid = self.getHost().id
+        return cmd
+
+
+    def getOobmDisableCmd(self):
+        cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd()
+        cmd.hostid = self.getHost().id
+        return cmd
+
+
+    def getOobmIssueActionCmd(self):
+        cmd = issueOutOfBandManagementPowerAction.issueOutOfBandManagementPowerActionCmd()
+        cmd.hostid = self.getHost().id
+        cmd.action = 'STATUS'
+        return cmd
+
+
+    def issuePowerActionCmd(self, action, timeout=None):
+        cmd = self.getOobmIssueActionCmd()
+        cmd.action = action
+        if timeout:
+            cmd.timeout = timeout
+        return self.apiclient.issueOutOfBandManagementPowerAction(cmd)
+
+
+    def configureAndEnableOobm(self):
+        self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
+        response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd())
+        self.assertEqual(response.enabled, True)
+
+
+    def startIpmiServer(self):
+        def startIpmiServer(tname, server):
+            self.debug("Starting ipmisim server")
+            try:
+                server.serve_forever()
+            except Exception: pass
+        IpmiServerContext('reset')
+        ThreadedIpmiServer.allow_reuse_address = False
+        server = ThreadedIpmiServer(('0.0.0.0', self.getIpmiServerPort()), IpmiServer)
+        thread.start_new_thread(startIpmiServer, ("ipmi-server", server,))
+        self.server = server
+
+
+    def checkSyncToState(self, state, interval):
+        self.debug("Waiting for background thread to update powerstate to " + state)
+        time.sleep(1 + int(interval)*2/1000) # interval is in ms
+        response = self.getHost(hostId=self.getHost().id).outofbandmanagement
+        self.assertEqual(response.powerstate, state)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_configure_invalid_driver(self):
+        """
+            Tests out-of-band management configuration with invalid driver
+        """
+        cmd = self.getOobmConfigCmd()
+        cmd.driver = 'randomDriverThatDoesNotExist'
+        try:
+            response = self.apiclient.configureOutOfBandManagement(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_configure_default_driver(self):
+        """
+            Tests out-of-band management configuration with valid data
+        """
+        cmd = self.getOobmConfigCmd()
+        response = self.apiclient.configureOutOfBandManagement(cmd)
+        self.assertEqual(response.hostid, cmd.hostid)
+        self.assertEqual(response.driver, cmd.driver)
+        self.assertEqual(response.address, cmd.address)
+        self.assertEqual(response.port, str(cmd.port))
+        self.assertEqual(response.username, cmd.username)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_enable_feature_invalid(self):
+        """
+            Tests out-of-band management host enable feature with
+            invalid options
+        """
+        cmd = self.getOobmEnableCmd()
+        cmd.hostid = -1
+        try:
+            response = self.apiclient.enableOutOfBandManagementForHost(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+        try:
+            cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
+            response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+        try:
+            cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
+            response = self.apiclient.enableOutOfBandManagementForZone(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_disable_feature_invalid(self):
+        """
+            Tests out-of-band management host disable feature with
+            invalid options
+        """
+        cmd = self.getOobmDisableCmd()
+        cmd.hostid = -1
+        try:
+            response = self.apiclient.disableOutOfBandManagementForHost(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+        try:
+            cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd()
+            response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+        try:
+            cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd()
+            response = self.apiclient.disableOutOfBandManagementForZone(cmd)
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_enable_feature_valid(self):
+        """
+            Tests out-of-band management host enable feature with
+            valid options
+        """
+        self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
+        cmd = self.getOobmEnableCmd()
+        response = self.apiclient.enableOutOfBandManagementForHost(cmd)
+        self.assertEqual(response.hostid, cmd.hostid)
+        self.assertEqual(response.enabled, True)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_disable_feature_valid(self):
+        """
+            Tests out-of-band management host disable feature with
+            valid options
+        """
+
+        self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
+        cmd = self.getOobmDisableCmd()
+        response = self.apiclient.disableOutOfBandManagementForHost(cmd)
+        self.assertEqual(response.hostid, cmd.hostid)
+        self.assertEqual(response.enabled, False)
+
+        response = self.getHost(hostId=cmd.hostid).outofbandmanagement
+        self.assertEqual(response.powerstate, 'Disabled')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_enabledisable_across_clusterzones(self):
+        """
+            Tests out-of-band management enable/disable feature at cluster
+            and zone level sequentially Zone > Cluster > Host
+        """
+        self.configureAndEnableOobm()
+        self.startIpmiServer()
+        bmc = IpmiServerContext().bmc
+        bmc.powerstate = 'off'
+
+        host = self.getHost()
+
+        # Disable at zone level
+        cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd()
+        cmd.zoneid = host.zoneid
+        response = self.apiclient.disableOutOfBandManagementForZone(cmd)
+
+        # Disable at cluster level
+        cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd()
+        cmd.clusterid = host.clusterid
+        response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
+
+        # Disable at host level
+        cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd()
+        cmd.hostid = host.id
+        response = self.apiclient.disableOutOfBandManagementForHost(cmd)
+
+        try:
+            self.issuePowerActionCmd('STATUS')
+            self.fail("Exception was expected, oobm is disabled at zone level")
+        except Exception: pass
+
+        # Enable at zone level
+        cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
+        cmd.zoneid = host.zoneid
+        response = self.apiclient.enableOutOfBandManagementForZone(cmd)
+
+        try:
+            self.issuePowerActionCmd('STATUS')
+            self.fail("Exception was expected, oobm is disabled at cluster level")
+        except Exception: pass
+
+        # Check background thread syncs state to Disabled
+        response = self.getHost(hostId=host.id).outofbandmanagement
+        self.assertEqual(response.powerstate, 'Disabled')
+        self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort())
+        interval = list_configurations(
+            self.apiclient,
+            name='outofbandmanagement.sync.interval'
+        )[0].value
+        self.checkSyncToState('Disabled', interval)
+
+        # Enable at cluster level
+        cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
+        cmd.clusterid = host.clusterid
+        response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
+
+        try:
+            self.issuePowerActionCmd('STATUS')
+            self.fail("Exception was expected, oobm is disabled at host level")
+        except Exception: pass
+
+        # Enable at host level
+        cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
+        cmd.hostid = host.id
+        response = self.apiclient.enableOutOfBandManagementForHost(cmd)
+
+        response = self.issuePowerActionCmd('STATUS')
+        self.assertEqual(response.powerstate, 'Off')
+
+
+    def configureAndStartIpmiServer(self, power_state=None):
+        """
+            Setup ipmisim and enable out-of-band management for host
+        """
+        self.configureAndEnableOobm()
+        self.startIpmiServer()
+        if power_state:
+            bmc = IpmiServerContext().bmc
+            bmc.powerstate = power_state
+
+
+    def assertIssueCommandState(self, command, expected):
+        """
+            Asserts power action result for a given power command
+        """
+        if command != 'STATUS':
+            self.issuePowerActionCmd(command)
+        response = self.issuePowerActionCmd('STATUS')
+        self.assertEqual(response.powerstate, expected)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_status(self):
+        """
+            Tests out-of-band management issue power action
+        """
+        self.configureAndStartIpmiServer(power_state='on')
+        self.assertIssueCommandState('STATUS', 'On')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_on(self):
+        """
+            Tests out-of-band management issue power on action
+        """
+        self.configureAndStartIpmiServer()
+        self.assertIssueCommandState('ON', 'On')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_off(self):
+        """
+            Tests out-of-band management issue power off action
+        """
+        self.configureAndStartIpmiServer()
+        self.assertIssueCommandState('OFF', 'Off')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_cycle(self):
+        """
+            Tests out-of-band management issue power cycle action
+        """
+        self.configureAndStartIpmiServer()
+        self.assertIssueCommandState('CYCLE', 'On')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_reset(self):
+        """
+            Tests out-of-band management issue power reset action
+        """
+        self.configureAndStartIpmiServer()
+        self.assertIssueCommandState('RESET', 'On')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_soft(self):
+        """
+            Tests out-of-band management issue power soft action
+        """
+        self.configureAndStartIpmiServer()
+        self.assertIssueCommandState('SOFT', 'Off')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_background_powerstate_sync(self):
+        """
+            Tests out-of-band management background powerstate sync
+        """
+        self.debug("Testing oobm background sync")
+        interval = list_configurations(
+            self.apiclient,
+            name='outofbandmanagement.sync.interval'
+        )[0].value
+
+        self.configureAndEnableOobm()
+        self.startIpmiServer()
+        bmc = IpmiServerContext().bmc
+
+        bmc.powerstate = 'on'
+        self.checkSyncToState('On', interval)
+
+        bmc.powerstate = 'off'
+        self.checkSyncToState('Off', interval)
+
+        self.server.shutdown()
+        self.server.server_close()
+
+        # Check for unknown state (ipmi server not reachable)
+        self.checkSyncToState('Unknown', interval)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_multiple_mgmt_server_ownership(self):
+        """
+            Tests out-of-band management ownership expiry across multi-mgmt server
+        """
+        self.configureAndEnableOobm()
+
+        cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion
+
+        currentMsHosts = []
+        mshosts = self.dbclient.execute("select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion))
+        if len(mshosts) > 0:
+            currentMsHosts = map(lambda row: row[0], mshosts)
+
+        # Inject fake ms host
+        self.dbclient.execute("insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion))
+
+        # Pass ownership to the fake ms id
+        self.dbclient.execute("update oobm set mgmt_server_id=%d where port=%d" % (self.getFakeMsId(), self.getIpmiServerPort()))
+
+        self.debug("Testing oobm background sync")
+        pingInterval = float(list_configurations(
+            self.apiclient,
+            name='ping.interval'
+        )[0].value)
+
+        pingTimeout = float(list_configurations(
+            self.apiclient,
+            name='ping.timeout'
+        )[0].value)
+
+
+        def removeFakeMgmtServer(fakeMsRunId):
+            rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId)
+            if len(rows) > 0:
+                self.debug("Mgmt server is now trying to contact the fake mgmt server")
+                self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId)
+                self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId)
+                return True, None
+            return False, None
+
+        def checkOobmOwnershipExpiry(serverPort, fakeMsId):
+            rows = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (serverPort))
+            if len(rows) > 0 and rows[0][0] != fakeMsId:
+                self.debug("Out-of-band management ownership expired as node was detected to be gone")
+                return True, None
+            return False, None
+
+        retry_interval = 1 + (pingInterval * pingTimeout / 10)
+
+        res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId())
+        if not res:
+            self.fail("Management server failed to turn down or remove fake mgmt server")
+
+        res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry, self.getIpmiServerPort(), self.getFakeMsId())
+        if not res:
+            self.fail("Management server failed to expire ownership of fenced peer")
+
+        self.debug("Testing oobm background sync should claim new ownership")
+        interval = list_configurations(
+            self.apiclient,
+            name='outofbandmanagement.sync.interval'
+        )[0].value
+
+        self.startIpmiServer()
+        bmc = IpmiServerContext().bmc
+        bmc.powerstate = 'on'
+
+        self.checkSyncToState('On', interval)
+
+        result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort()))
+        newOwnerId = result[0][0]
+        self.assertTrue(newOwnerId in currentMsHosts)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_zchange_password(self):
+        """
+            Tests out-of-band management change password feature
+        """
+        self.configureAndEnableOobm()
+        self.startIpmiServer()
+
+        self.debug("Testing oobm change password")
+
+        cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd()
+        cmd.hostid = self.getHost().id
+        cmd.password = "Password12345"
+        response = self.apiclient.changeOutOfBandManagementPassword(cmd)
+        self.assertEqual(response.status, True)
+
+        bmc = IpmiServerContext().bmc
+        bmc.powerstate = 'on'
+        response = self.issuePowerActionCmd('STATUS')
+        self.assertEqual(response.status, True)
+        self.assertEqual(response.powerstate, 'On')
+
+        # Reset configuration, resets password
+        self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
+        self.assertEqual(response.status, True)
+
+        alerts = Alert.list(self.apiclient, keyword="auth-error",
+                        listall=True)
+        alertCount = 0
+        if alerts:
+            alertCount = len(alerts)
+
+        try:
+            response = self.issuePowerActionCmd('STATUS')
+            self.fail("Expected an exception to be thrown, failing")
+        except Exception: pass
+
+        alerts = Alert.list(self.apiclient, keyword="auth-error",
+                        listall=True)
+
+        # At least one alert was sent
+        self.assertTrue((len(alerts) - alertCount) > 0)

tools/apidoc/gen_toc.py

@@ -61,6 +61,7 @@ known_categories = {
     'StaticNat': 'NAT',
     'IpForwarding': 'NAT',
     'Host': 'Host',
+    'OutOfBand': 'Out-of-band Management',
     'Cluster': 'Cluster',
     'Account': 'Account',
     'Role': 'Role',

tools/marvin/setup.py

@@ -52,7 +52,8 @@ setup(name="Marvin",
           "nose >= 1.3.3",
           "ddt >= 0.4.0",
           "pyvmomi >= 5.5.0",
-          "netaddr >= 0.7.14"
+          "netaddr >= 0.7.14",
+          "ipmisim >= 0.7"
       ],
       py_modules=['marvin.marvinPlugin'],
       zip_safe=False,

tools/travis/before_install.sh

@@ -86,10 +86,19 @@ sudo service mysql restart
 echo -e "\nInstalling Development tools: "
 RETRY_COUNT=3
 
-sudo apt-get -q -y install uuid-runtime genisoimage netcat > /dev/null
+sudo apt-get -q -y install uuid-runtime genisoimage netcat freeipmi-common freeipmi-tools libfreeipmi12 > /dev/null
 if [[ $? -ne 0 ]]; then
   echo -e "\napt-get packages failed to install"
 fi
+
+# We need version 1.8.15 or above, default installed version is buggy
+wget http://mirrors.kernel.org/ubuntu/pool/universe/i/ipmitool/ipmitool_1.8.15-1ubuntu1.1_amd64.deb -O /tmp/ipmitool.deb
+if [[ $? -eq 0 ]]; then
+  sudo dpkg -i /tmp/ipmitool.deb
+  sudo apt-get install -f -y
+  ipmitool -V
+fi
+
 echo "<settings>
   <mirrors>
     <mirror>
@@ -107,7 +116,7 @@ pip install --user --upgrade pip
 
 for ((i=0;i<$RETRY_COUNT;i++))
 do
-  pip install --user --upgrade lxml paramiko nose texttable > /tmp/piplog
+  pip install --user --upgrade lxml paramiko nose texttable ipmisim > /tmp/piplog
   if [[ $? -eq 0 ]]; then
     echo -e "\npython packages installed successfully"
     break;

ui/css/cloudstack3.css

@@ -12703,6 +12703,54 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it
   background-position: -137px -614px;
 }
 
+.blankOutOfBandManagement .icon {
+  background-position: -266px -31px;
+}
+
+.blankOutOfBandManagement:hover .icon {
+  background-position: -266px -31px;
+}
+
+.configureOutOfBandManagement .icon {
+  background-position: -168px -31px;
+}
+
+.configureOutOfBandManagement:hover .icon {
+  background-position: -168px -613px;
+}
+
+.enableOutOfBandManagement .icon {
+  background-position: -138px -65px;
+}
+
+.enableOutOfBandManagement:hover .icon {
+  background-position: -138px -647px;
+}
+
+.disableOutOfBandManagement .icon {
+  background-position: -138px -123px;
+}
+
+.disableOutOfBandManagement:hover .icon {
+  background-position: -138px -705px;
+}
+
+.issueOutOfBandManagementPowerAction .icon {
+  background-position: -266px -3px;
+}
+
+.issueOutOfBandManagementPowerAction:hover .icon {
+  background-position: -265px -584px;
+}
+
+.changeOutOfBandManagementPassword .icon {
+  background-position: -68px -30px;
+}
+
+.changeOutOfBandManagementPassword:hover .icon {
+  background-position: -68px -612px;
+}
+
 .enableMaintenanceMode .icon {
   background-position: -138px -65px;
 }

ui/dictionary.jsp

@@ -871,6 +871,7 @@ dictionary = {
 'label.metrics.property': '<fmt:message key="label.metrics.property" />',
 'label.metrics.scope': '<fmt:message key="label.metrics.scope" />',
 'label.metrics.state': '<fmt:message key="label.metrics.state" />',
+'label.metrics.outofbandmanagementpowerstate': '<fmt:message key="label.metrics.outofbandmanagementpowerstate" />',
 'label.metrics.storagepool': '<fmt:message key="label.metrics.storagepool" />',
 'label.metrics.vm.name': '<fmt:message key="label.metrics.vm.name" />',
 'label.migrate.instance.to': '<fmt:message key="label.migrate.instance.to" />',
@@ -987,6 +988,26 @@ dictionary = {
 'label.port.forwarding': '<fmt:message key="label.port.forwarding" />',
 'label.port.forwarding.policies': '<fmt:message key="label.port.forwarding.policies" />',
 'label.port.range': '<fmt:message key="label.port.range" />',
+'label.powerstate': '<fmt:message key="label.powerstate" />',
+'label.outofbandmanagement': '<fmt:message key="label.outofbandmanagement" />',
+'label.outofbandmanagement.action.issue': '<fmt:message key="label.outofbandmanagement.action.issue" />',
+'label.outofbandmanagement.action': '<fmt:message key="label.outofbandmanagement.action" />',
+'label.outofbandmanagement.address': '<fmt:message key="label.outofbandmanagement.address" />',
+'label.outofbandmanagement.changepassword': '<fmt:message key="label.outofbandmanagement.changepassword" />',
+'label.outofbandmanagement.configure': '<fmt:message key="label.outofbandmanagement.configure" />',
+'label.outofbandmanagement.driver': '<fmt:message key="label.outofbandmanagement.driver" />',
+'label.outofbandmanagement.disable': '<fmt:message key="label.outofbandmanagement.disable" />',
+'label.outofbandmanagement.enable': '<fmt:message key="label.outofbandmanagement.enable" />',
+'label.outofbandmanagement.password': '<fmt:message key="label.outofbandmanagement.password" />',
+'label.outofbandmanagement.port': '<fmt:message key="label.outofbandmanagement.port" />',
+'label.outofbandmanagement.timeout': '<fmt:message key="label.outofbandmanagement.timeout" />',
+'label.outofbandmanagement.username': '<fmt:message key="label.outofbandmanagement.username" />',
+'message.outofbandmanagement.changepassword': '<fmt:message key="message.outofbandmanagement.changepassword" />',
+'message.outofbandmanagement.configure': '<fmt:message key="message.outofbandmanagement.configure" />',
+'message.outofbandmanagement.disable': '<fmt:message key="message.outofbandmanagement.disable" />',
+'message.outofbandmanagement.enable': '<fmt:message key="message.outofbandmanagement.enable" />',
+'message.outofbandmanagement.issue': '<fmt:message key="message.outofbandmanagement.issue" />',
+'message.outofbandmanagement.action.maintenance': '<fmt:message key="message.outofbandmanagement.action.maintenance" />',
 'label.PreSetup': '<fmt:message key="label.PreSetup" />',
 'label.prev': '<fmt:message key="label.prev" />',
 'label.previous': '<fmt:message key="label.previous" />',

ui/scripts/metrics.js

@@ -550,6 +550,19 @@
                     },
                     compact: true
                 },
+                outofbandmanagementpowerstate: {
+                    label: 'label.metrics.outofbandmanagementpowerstate',
+                    converter: function (str) {
+                        // For localization
+                        return str;
+                    },
+                    indicator: {
+                        'On': 'on',
+                        'Off': 'off',
+                        'Unknown': 'warning'
+                    },
+                    compact: true
+                },
                 instances: {
                     label: 'label.instances'
                 },
@@ -633,6 +646,9 @@
                         var items = json.listhostsresponse.host;
                         if (items) {
                             $.each(items, function(idx, host) {
+                                if (host && host.outofbandmanagement) {
+                                    items[idx].outofbandmanagementpowerstate = host.outofbandmanagement.powerstate;
+                                }
                                 items[idx].cores = host.cpunumber;
                                 items[idx].cputotal = (parseFloat(host.cpunumber) * parseFloat(host.cpuspeed) / 1000.0).toFixed(2);
                                 if (host.cpuused) {

ui/scripts/system.js

@@ -8241,6 +8241,82 @@
                                                 }
                                             });
                                         }
+                                    },
+                                    enableOutOfBandManagement: {
+                                        label: 'label.outofbandmanagement.enable',
+                                        action: function (args) {
+                                            var data = {
+                                                zoneid: args.context.physicalResources[0].id
+                                            };
+                                            $.ajax({
+                                                url: createURL("enableOutOfBandManagementForZone"),
+                                                data: data,
+                                                success: function (json) {
+                                                    var jid = json.enableoutofbandmanagementforzoneresponse.jobid;
+                                                    args.response.success({
+                                                        _custom: {
+                                                            jobId: jid,
+                                                            getActionFilter: function () {
+                                                                return zoneActionfilter;
+                                                            }
+                                                        }
+                                                    });
+                                                },
+                                                error: function (json) {
+                                                    args.response.error(parseXMLHttpResponse(json));
+                                                }
+
+                                            });
+                                        },
+                                        messages: {
+                                            confirm: function (args) {
+                                                return 'message.outofbandmanagement.enable';
+                                            },
+                                            notification: function (args) {
+                                                return 'message.outofbandmanagement.enable';
+                                            }
+                                        },
+                                        notification: {
+                                            poll: pollAsyncJobResult
+                                        }
+                                    },
+                                    disableOutOfBandManagement: {
+                                        label: 'label.outofbandmanagement.disable',
+                                        action: function (args) {
+                                            var data = {
+                                                zoneid: args.context.physicalResources[0].id
+                                            };
+                                            $.ajax({
+                                                url: createURL("disableOutOfBandManagementForZone"),
+                                                data: data,
+                                                success: function (json) {
+                                                    var jid = json.disableoutofbandmanagementforzoneresponse.jobid;
+                                                    args.response.success({
+                                                        _custom: {
+                                                            jobId: jid,
+                                                            getActionFilter: function () {
+                                                                return zoneActionfilter;
+                                                            }
+                                                        }
+                                                    });
+                                                },
+                                                error: function (json) {
+                                                    args.response.error(parseXMLHttpResponse(json));
+                                                }
+
+                                            });
+                                        },
+                                        messages: {
+                                            confirm: function (args) {
+                                                return 'message.outofbandmanagement.disable';
+                                            },
+                                            notification: function (args) {
+                                                return 'message.outofbandmanagement.disable';
+                                            }
+                                        },
+                                        notification: {
+                                            poll: pollAsyncJobResult
+                                        }
                                     }
                                 },
                                 tabs: {
@@ -9058,8 +9134,17 @@
                                         url: createURL('listHosts'),
                                         data: data,
                                         success: function (json) {
+                                            var items = json.listhostsresponse.host;
+                                            if (items) {
+                                                $.each(items, function(idx, host) {
+                                                    if (host && host.outofbandmanagement) {
+                                                        items[idx].powerstate = host.outofbandmanagement.powerstate;
+                                                    }
+                                                });
+                                            }
+
                                             args.response.success({
-                                                data: json.listhostsresponse.host
+                                                data: items
                                             });
                                         },
                                         error: function (json) {
@@ -14970,7 +15055,85 @@
                                         args.complete();
                                     }
                                 }
+                            },
+
+                            enableOutOfBandManagement: {
+                                label: 'label.outofbandmanagement.enable',
+                                action: function (args) {
+                                    var data = {
+                                        clusterid: args.context.clusters[0].id,
+                                    };
+                                    $.ajax({
+                                        url: createURL("enableOutOfBandManagementForCluster"),
+                                        data: data,
+                                        success: function (json) {
+                                            var jid = json.enableoutofbandmanagementforclusterresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return clusterActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+                                    });
+                                },
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.enable';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.enable';
+                                    }
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
+                            },
+
+                            disableOutOfBandManagement: {
+                                label: 'label.outofbandmanagement.disable',
+                                action: function (args) {
+                                    var data = {
+                                        clusterid: args.context.clusters[0].id,
+                                    };
+                                    $.ajax({
+                                        url: createURL("disableOutOfBandManagementForCluster"),
+                                        data: data,
+                                        success: function (json) {
+                                            var jid = json.disableoutofbandmanagementforclusterresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return clusterActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+
+                                    });
+                                },
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.disable';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.disable';
+                                    }
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
                             }
+
                         },
 
                         tabs: {
@@ -15328,7 +15491,15 @@
                                 'Alert': 'off',
                                 'Error': 'off'
                             }
-                        }
+                        },
+                        powerstate: {
+                            label: 'label.powerstate',
+                            indicator: {
+                                'On': 'on',
+                                'Off': 'off',
+                                'Unknown': 'warning'
+                            },
+                        },
                     },
 
                     dataProvider: function (args) {
@@ -15355,13 +15526,20 @@
                             //Instances menu > Instance detailView > View Hosts
                             array1.push("&id=" + args.context.instances[0].hostid);
                         }
-
                         $.ajax({
                             url: createURL("listHosts&type=Routing" + array1.join("") + "&page=" + args.page + "&pagesize=" + pageSize),
                             dataType: "json",
                             async: true,
                             success: function (json) {
                                 var items = json.listhostsresponse.host;
+                                if (items) {
+                                    $.each(items, function(idx, host) {
+                                        if (host && host.outofbandmanagement) {
+                                            items[idx].powerstate = host.outofbandmanagement.powerstate;
+                                        }
+                                    });
+                                }
+
                                 args.response.success({
                                     actionFilter: hostActionfilter,
                                     data: items
@@ -15998,6 +16176,7 @@
                                 }
                             },
 
+
                             dedicate: {
                                 label: 'label.dedicate.host',
                                 messages: {
@@ -16347,13 +16526,331 @@
                                         args.complete();
                                     }
                                 }
+                            },
+
+                            blankOutOfBandManagement: {
+                                label: '',
+                                action: function (args) {
+                                }
+                            },
+
+                            configureOutOfBandManagement: {
+                                label: 'label.outofbandmanagement.configure',
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.configure';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.configure';
+                                    }
+                                },
+                                createForm: {
+                                    title: 'label.outofbandmanagement.configure',
+                                    fields: {
+                                        address: {
+                                            label: 'label.outofbandmanagement.address',
+                                            validation: {
+                                                required: true
+                                            }
+                                        },
+                                        port: {
+                                            label: 'label.outofbandmanagement.port',
+                                            validation: {
+                                                required: true
+                                            }
+                                        },
+                                        username: {
+                                            label: 'label.outofbandmanagement.username',
+                                            validation: {
+                                                required: false
+                                            }
+                                        },
+                                        password: {
+                                            label: 'label.outofbandmanagement.password',
+                                            isPassword: true,
+                                            validation: {
+                                                required: false
+                                            },
+                                        },
+                                        driver: {
+                                            label: 'label.outofbandmanagement.driver',
+                                            validation: {
+                                                required: true
+                                            },
+                                            select: function (args) {
+                                                var oobm = args.context.hosts[0].outofbandmanagement;
+                                                if (oobm) {
+                                                    args.$form.find('input[name=address]').val(oobm.address);
+                                                    args.$form.find('input[name=port]').val(oobm.port);
+                                                    args.$form.find('input[name=username]').val(oobm.username);
+
+                                                    args.$form.find('input[name=address]').change(function() {
+                                                        $this.find('input[name=address]').val(oobm.address);
+                                                    });
+                                                }
+
+                                                var items = [];
+                                                items.push({
+                                                    id: 'ipmitool',
+                                                    description: 'ipmitool - ipmitool based shell driver'
+                                                });
+                                                args.response.success({
+                                                    data: items
+                                                });
+                                            }
+                                        }
+                                    }
+                                },
+                                action: function (args) {
+                                    var data = args.data;
+                                    data.hostid = args.context.hosts[0].id;
+
+                                    $.ajax({
+                                        url: createURL('configureOutOfBandManagement'),
+                                        data: data,
+                                        dataType: 'json',
+                                        success: function (json) {
+                                            var response = json.configureoutofbandmanagementresponse;
+                                            args.response.success({
+                                                actionFilter: hostActionfilter,
+                                                data: response
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+                                    });
+                                },
+                                notification: {
+                                    poll: function (args) {
+                                        args.complete();
+                                    }
+                                }
+                            },
+
+                            enableOutOfBandManagement: {
+                                label: 'label.outofbandmanagement.enable',
+                                action: function (args) {
+                                    var data = {
+                                        hostid: args.context.hosts[0].id,
+                                    };
+                                    $.ajax({
+                                        url: createURL("enableOutOfBandManagementForHost"),
+                                        data: data,
+                                        success: function (json) {
+                                            var jid = json.enableoutofbandmanagementforhostresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return hostActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+
+                                    });
+                                },
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.enable';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.enable';
+                                    }
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
+                            },
+
+                            disableOutOfBandManagement: {
+                                label: 'label.outofbandmanagement.disable',
+                                action: function (args) {
+                                    var data = {
+                                        hostid: args.context.hosts[0].id,
+                                    };
+                                    $.ajax({
+                                        url: createURL("disableOutOfBandManagementForHost"),
+                                        data: data,
+                                        success: function (json) {
+                                            var jid = json.disableoutofbandmanagementforhostresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return hostActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+
+                                    });
+                                },
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.disable';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.disable';
+                                    }
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
+                            },
+
+                            issueOutOfBandManagementPowerAction: {
+                                label: 'label.outofbandmanagement.action.issue',
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.issue';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.issue';
+                                    }
+                                },
+                                createForm: {
+                                    title: 'label.outofbandmanagement.action.issue',
+                                    desc: function(args) {
+                                          var host = args.context.hosts[0];
+                                          if (host.resourcestate == 'Maintenance' || host.resourcestate == 'PrepareForMaintenance' || host.resourcestate == 'ErrorInMaintenance') {
+                                              return _l('message.outofbandmanagement.action.maintenance');
+                                          }
+                                    },
+                                    fields: {
+                                        action: {
+                                            label: 'label.outofbandmanagement.action',
+                                            validation: {
+                                                required: true
+                                            },
+                                            select: function (args) {
+                                                var items = [];
+                                                items.push({
+                                                    id: 'ON',
+                                                    description: 'ON - turn on host'
+                                                });
+                                                items.push({
+                                                    id: 'OFF',
+                                                    description: 'OFF - turn off host'
+                                                });
+                                                items.push({
+                                                    id: 'CYCLE',
+                                                    description: 'CYCLE - power cycle the host'
+                                                });
+                                                items.push({
+                                                    id: 'RESET',
+                                                    description: 'RESET - power reset the host'
+                                                });
+                                                items.push({
+                                                    id: 'SOFT',
+                                                    description: 'SOFT - soft shutdown the host using ACPI etc'
+                                                });
+                                                items.push({
+                                                    id: 'STATUS',
+                                                    description: 'STATUS - update power status of the host'
+                                                });
+                                                args.response.success({
+                                                    data: items
+                                                });
+                                            }
+                                        },
+                                    }
+                                },
+                                action: function (args) {
+                                    var data = args.data;
+                                    data.hostid = args.context.hosts[0].id;
+                                    $.ajax({
+                                        url: createURL('issueOutOfBandManagementPowerAction'),
+                                        data: data,
+                                        dataType: 'json',
+                                        success: function (json) {
+                                            var jid = json.issueoutofbandmanagementpoweractionresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return hostActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+                                    });
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
+                            },
+
+                            changeOutOfBandManagementPassword: {
+                                label: 'label.outofbandmanagement.changepassword',
+                                messages: {
+                                    confirm: function (args) {
+                                        return 'message.outofbandmanagement.changepassword';
+                                    },
+                                    notification: function (args) {
+                                        return 'message.outofbandmanagement.changepassword';
+                                    }
+                                },
+                                createForm: {
+                                    title: 'label.outofbandmanagement.changepassword',
+                                    fields: {
+                                        password: {
+                                            label: 'label.outofbandmanagement.password',
+                                            isPassword: true,
+                                            validation: {
+                                                required: false
+                                            },
+                                        },
+                                    }
+                                },
+                                action: function (args) {
+                                    var data = args.data;
+                                    data.hostid = args.context.hosts[0].id;
+                                    $.ajax({
+                                        url: createURL('changeOutOfBandManagementPassword'),
+                                        data: data,
+                                        dataType: 'json',
+                                        success: function (json) {
+                                            var jid = json.changeoutofbandmanagementpasswordresponse.jobid;
+                                            args.response.success({
+                                                _custom: {
+                                                    jobId: jid,
+                                                    getActionFilter: function () {
+                                                        return hostActionfilter;
+                                                    }
+                                                }
+                                            });
+                                        },
+                                        error: function (json) {
+                                            args.response.error(parseXMLHttpResponse(json));
+                                        }
+                                    });
+                                },
+                                notification: {
+                                    poll: pollAsyncJobResult
+                                }
                             }
+
                         },
                         tabFilter: function (args) {
                             var hiddenTabs =[];
-                            if (args.context.hosts[0].gpugroup == null) {
+                            var host = args.context.hosts[0];
+                            if (host.gpugroup == null) {
                                 hiddenTabs.push("gpu");
                             }
+                            if (host.outofbandmanagement == null || !host.outofbandmanagement.enabled) {
+                                hiddenTabs.push("outofbandmanagement");
+                            }
                             return hiddenTabs;
                         },
                         tabs: {
@@ -16390,6 +16887,9 @@
                                     state: {
                                         label: 'label.state'
                                     },
+                                    powerstate: {
+                                        label: 'label.powerstate'
+                                    },
                                     type: {
                                         label: 'label.type'
                                     },
@@ -16501,6 +17001,10 @@
                                         async: true,
                                         success: function (json) {
                                             var item = json.listhostsresponse.host[0];
+                                            if (item && item.outofbandmanagement) {
+                                                item.powerstate = item.outofbandmanagement.powerstate;
+                                            }
+
                                             $.ajax({
                                                 url: createURL("listDedicatedHosts&hostid=" + args.context.hosts[0].id),
                                                 dataType: "json",
@@ -16532,6 +17036,44 @@
                                 }
                             },
 
+                            outofbandmanagement: {
+                                title: 'label.outofbandmanagement',
+                                fields: {
+                                    powerstate: {
+                                        label: 'label.powerstate'
+                                    },
+                                    driver: {
+                                        label: 'label.outofbandmanagement.driver'
+                                    },
+                                    username: {
+                                        label: 'label.outofbandmanagement.username'
+                                    },
+                                    address: {
+                                        label: 'label.outofbandmanagement.address'
+                                    },
+                                    port: {
+                                        label: 'label.outofbandmanagement.port'
+                                    }
+                                },
+                                dataProvider: function (args) {
+                                    $.ajax({
+                                        url: createURL("listHosts&id=" + args.context.hosts[0].id),
+                                        dataType: "json",
+                                        async: true,
+                                        success: function (json) {
+                                            var host = json.listhostsresponse.host[0];
+                                            var oobm = {};
+                                            if (host && host.outofbandmanagement) {
+                                                oobm = host.outofbandmanagement;
+                                            }
+                                            args.response.success({
+                                                data: oobm
+                                            });
+                                        }
+                                    });
+                                }
+                            },
+
                             stats: {
                                 title: 'label.statistics',
                                 fields: {
@@ -20807,6 +21349,13 @@
         allowedActions.push("disable");
 
         allowedActions.push("remove");
+
+        if (jsonObj.hasOwnProperty('resourcedetails') && jsonObj['resourcedetails'].hasOwnProperty('outOfBandManagementEnabled') && jsonObj['resourcedetails']['outOfBandManagementEnabled'] == 'false') {
+            allowedActions.push("enableOutOfBandManagement");
+        } else {
+            allowedActions.push("disableOutOfBandManagement");
+        }
+
         return allowedActions;
     }
 
@@ -20892,6 +21441,12 @@
 
         allowedActions.push("remove");
 
+        if (jsonObj.hasOwnProperty('resourcedetails') && jsonObj['resourcedetails'].hasOwnProperty('outOfBandManagementEnabled') && jsonObj['resourcedetails']['outOfBandManagementEnabled'] == 'false') {
+            allowedActions.push("enableOutOfBandManagement");
+        } else {
+            allowedActions.push("disableOutOfBandManagement");
+        }
+
         return allowedActions;
     }
 
@@ -20928,6 +21483,16 @@
             allowedActions.push("remove");
         }
 
+        allowedActions.push("blankOutOfBandManagement");
+        allowedActions.push("configureOutOfBandManagement");
+        if (jsonObj.hasOwnProperty("outofbandmanagement") && jsonObj.outofbandmanagement.enabled) {
+            allowedActions.push("issueOutOfBandManagementPowerAction");
+            allowedActions.push("changeOutOfBandManagementPassword");
+            allowedActions.push("disableOutOfBandManagement");
+        } else {
+            allowedActions.push("enableOutOfBandManagement");
+        }
+
         if ((jsonObj.state == "Down" || jsonObj.state == "Alert" || jsonObj.state == "Disconnected") && ($.inArray("remove", allowedActions) == -1)) {
             allowedActions.push("remove");
         }

utils/src/main/java/com/cloud/utils/fsm/StateMachine2.java

@@ -45,10 +45,20 @@ public class StateMachine2<S, E, V extends StateObject<S>> {
 
     }
 
+    public void addInitialTransition(E event, S toState) {
+        addTransition(null, event, toState);
+    }
+
     public void addTransition(S currentState, E event, S toState) {
       addTransition(new Transition<S, E>(currentState, event, toState, null));
     }
 
+    @SafeVarargs
+    public final void addTransitionFromStates(E event, S toState, S... fromStates) {
+        for (S fromState : fromStates) {
+            addTransition(fromState, event, toState);
+        }
+    }
 
     public void addTransition(Transition<S, E> transition) {
       S currentState = transition.getCurrentState();

utils/src/main/java/org/apache/cloudstack/utils/process/ProcessResult.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.utils.process;
+
+public final class ProcessResult {
+    private final String stdOutput;
+    private final String stdError;
+    private final int returnCode;
+
+    public ProcessResult(String stdOutput, String stdError, int returnCode) {
+        this.stdOutput = stdOutput;
+        this.stdError = stdError;
+        this.returnCode = returnCode;
+    }
+
+    public String getStdOutput() {
+        return stdOutput;
+    }
+
+    public String getStdError() {
+        return stdError;
+    }
+
+    public int getReturnCode() {
+        return returnCode;
+    }
+
+    public boolean isSuccess() {
+        return returnCode == 0;
+    }
+}

utils/src/main/java/org/apache/cloudstack/utils/process/ProcessRunner.java

@@ -0,0 +1,115 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.utils.process;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.io.CharStreams;
+import org.apache.log4j.Logger;
+import org.joda.time.Duration;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public final class ProcessRunner {
+    public static final Logger LOG = Logger.getLogger(ProcessRunner.class);
+
+    // Default maximum timeout of 5 minutes for any command
+    public static final Duration DEFAULT_MAX_TIMEOUT = new Duration(5 * 60 * 1000);
+    private final ExecutorService executor;
+
+    public ProcessRunner(ExecutorService executor) {
+        this.executor = executor;
+    }
+
+    /**
+     * Executes a process with provided list of commands with a max default timeout
+     * of 5 minutes
+     * @param commands list of string commands
+     * @return returns process result
+     */
+    public ProcessResult executeCommands(final List<String> commands) {
+        return executeCommands(commands, DEFAULT_MAX_TIMEOUT);
+    }
+
+    /**
+     * Executes a process with provided list of commands with a given timeout that is less
+     * than or equal to DEFAULT_MAX_TIMEOUT
+     * @param commands list of string commands
+     * @param timeOut timeout duration
+     * @return returns process result
+     */
+    public ProcessResult executeCommands(final List<String> commands, final Duration timeOut) {
+        Preconditions.checkArgument(commands != null && timeOut != null
+                && timeOut.getStandardSeconds() > 0L
+                && (timeOut.compareTo(DEFAULT_MAX_TIMEOUT) <= 0)
+                && executor != null);
+
+        int retVal = -2;
+        String stdOutput = null;
+        String stdError = null;
+
+        try {
+            final Process process = new ProcessBuilder().command(commands).start();
+            final Future<Integer> processFuture = executor.submit(new Callable<Integer>() {
+                @Override
+                public Integer call() throws Exception {
+                    return process.waitFor();
+                }
+            });
+            try {
+                retVal = processFuture.get(timeOut.getStandardSeconds(), TimeUnit.SECONDS);
+            } catch (ExecutionException e) {
+                retVal = -2;
+                stdError = e.getMessage();
+                if (LOG.isTraceEnabled()) {
+                    LOG.trace("Failed to complete the requested command due to execution error: " + e.getMessage());
+                }
+            } catch (TimeoutException e) {
+                retVal = -1;
+                stdError = "Operation timed out, aborted";
+                if (LOG.isTraceEnabled()) {
+                    LOG.trace("Failed to complete the requested command within timeout: " + e.getMessage());
+                }
+            } finally {
+                if (Strings.isNullOrEmpty(stdError)) {
+                    stdOutput = CharStreams.toString(new InputStreamReader(process.getInputStream()));
+                    stdError = CharStreams.toString(new InputStreamReader(process.getErrorStream()));
+                }
+                process.destroy();
+            }
+            if (LOG.isTraceEnabled()) {
+                LOG.trace("Process standard output: " + stdOutput);
+                LOG.trace("Process standard error output: " + stdError);
+            }
+        } catch (IOException | InterruptedException e) {
+            stdError = e.getMessage();
+            LOG.error("Exception caught error running commands: " + e.getMessage());
+        }
+        return new ProcessResult(stdOutput, stdError, retVal);
+    }
+}

utils/src/test/java/org/apache/cloudstack/utils/process/ProcessTest.java

@@ -0,0 +1,69 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.utils.process;
+
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.google.common.base.Strings;
+import org.joda.time.Duration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProcessTest {
+
+    private static final ExecutorService executor = Executors.newFixedThreadPool(10, new NamedThreadFactory("IpmiToolDriverTest"));
+    private static final ProcessRunner RUNNER = new ProcessRunner(executor);
+
+    @Test
+    public void testProcessRunner() {
+        ProcessResult result = RUNNER.executeCommands(Arrays.asList("ls", "/tmp"));
+        Assert.assertEquals(result.getReturnCode(), 0);
+        Assert.assertTrue(Strings.isNullOrEmpty(result.getStdError()));
+        Assert.assertTrue(result.getStdOutput().length() > 0);
+    }
+
+    @Test
+    public void testProcessRunnerWithTimeout() {
+        ProcessResult result = RUNNER.executeCommands(Arrays.asList("sleep", "5"), Duration.standardSeconds(1));
+        Assert.assertNotEquals(result.getReturnCode(), 0);
+        Assert.assertTrue(result.getStdError().length() > 0);
+        Assert.assertEquals(result.getStdError(), "Operation timed out, aborted");
+    }
+
+    @Test
+    public void testProcessRunnerWithTimeoutAndException() {
+        ProcessResult result = RUNNER.executeCommands(Arrays.asList("ls", "/some/dir/that/should/not/exist"), Duration.standardSeconds(2));
+        Assert.assertNotEquals(result.getReturnCode(), 0);
+        Assert.assertTrue(result.getStdError().length() > 0);
+        Assert.assertNotEquals(result.getStdError(), "Operation timed out, aborted");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testProcessRunnerWithMoreThanMaxAllowedTimeout() {
+        RUNNER.executeCommands(Arrays.asList("ls", "/some/dir/that/should/not/exist"), ProcessRunner.DEFAULT_MAX_TIMEOUT.plus(1000));
+        Assert.fail("Illegal argument exception was expected");
+    }
+}