CLOUDSTACK-9782: Nested-oobm CloudStack plugin

Nested out-of-band management plugin to work with hosts that are VMs in
a CloudStack env.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.travis.yml

@@ -64,6 +64,7 @@ env:
              smoke/test_non_contigiousvlan"
 
     - TESTS="smoke/test_outofbandmanagement
+             smoke/test_outofbandmanagement_nestedplugin
              smoke/test_over_provisioning
              smoke/test_password_server
              smoke/test_portable_publicip

api/src/org/apache/cloudstack/api/response/OutOfBandManagementResponse.java

@@ -91,7 +91,7 @@ public class OutOfBandManagementResponse extends BaseResponse {
         this.setDriver(outOfBandManagementConfig.getDriver());
         this.setIpAddress(outOfBandManagementConfig.getAddress());
         if (outOfBandManagementConfig.getPort() != null) {
-            this.setPort(String.valueOf(outOfBandManagementConfig.getPort()));
+            this.setPort(outOfBandManagementConfig.getPort());
         }
         this.setUsername(outOfBandManagementConfig.getUsername());
         if (!Strings.isNullOrEmpty(outOfBandManagementConfig.getPassword())) {

api/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagement.java

@@ -39,7 +39,7 @@ public interface OutOfBandManagement extends StateObject<OutOfBandManagement.Pow
 
     String getAddress();
 
-    Integer getPort();
+    String getPort();
 
     String getUsername();
 
@@ -53,7 +53,7 @@ public interface OutOfBandManagement extends StateObject<OutOfBandManagement.Pow
 
     void setAddress(String address);
 
-    void setPort(Integer port);
+    void setPort(String port);
 
     void setUsername(String username);
 

client/pom.xml

@@ -264,6 +264,11 @@
       <artifactId>cloud-plugin-outofbandmanagement-driver-ipmitool</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-plugin-outofbandmanagement-driver-nested-cloudstack</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-mom-rabbitmq</artifactId>

engine/schema/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementVO.java

@@ -59,7 +59,7 @@ public class OutOfBandManagementVO implements OutOfBandManagement {
     private String address;
 
     @Column(name = "port")
-    private Integer port;
+    private String port;
 
     @Column(name = "username")
     private String username;
@@ -121,7 +121,7 @@ public class OutOfBandManagementVO implements OutOfBandManagement {
     }
 
     @Override
-    public Integer getPort() {
+    public String getPort() {
         return port;
     }
 
@@ -173,7 +173,7 @@ public class OutOfBandManagementVO implements OutOfBandManagement {
     }
 
     @Override
-    public void setPort(Integer port) {
+    public void setPort(String port) {
         this.port = port;
     }
 

plugins/outofbandmanagement-drivers/nested-cloudstack/pom.xml

@@ -0,0 +1,46 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>cloud-plugin-outofbandmanagement-driver-nested-cloudstack</artifactId>
+  <name>Apache CloudStack Plugin - Power Management Driver nested-cloudstack</name>
+  <parent>
+    <groupId>org.apache.cloudstack</groupId>
+    <artifactId>cloudstack-plugins</artifactId>
+    <version>4.11.0.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>br.com.autonomiccs</groupId>
+        <artifactId>apache-cloudstack-java-client</artifactId>
+        <version>1.0.4</version>
+    </dependency>
+  </dependencies>
+</project>

plugins/outofbandmanagement-drivers/nested-cloudstack/resources/META-INF/cloudstack/nested-cloudstack/module.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+name=nested-cloudstack
+parent=outofbandmanagement

plugins/outofbandmanagement-drivers/nested-cloudstack/resources/META-INF/cloudstack/nested-cloudstack/spring-nested-cloudstack-context.xml

@@ -0,0 +1,29 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
+>
+
+    <bean id="nestedCloudStackOutOfBandManagementDriver" class="org.apache.cloudstack.outofbandmanagement.driver.nestedcloudstack.NestedCloudStackOutOfBandManagementDriver">
+        <property name="name" value="NESTEDCLOUDSTACK" />
+    </bean>
+
+</beans>

plugins/outofbandmanagement-drivers/nested-cloudstack/src/org/apache/cloudstack/outofbandmanagement/driver/nestedcloudstack/NestedCloudStackOutOfBandManagementDriver.java

@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.outofbandmanagement.driver.nestedcloudstack;
+
+import br.com.autonomiccs.apacheCloudStack.client.ApacheCloudStackClient;
+import br.com.autonomiccs.apacheCloudStack.client.ApacheCloudStackRequest;
+import br.com.autonomiccs.apacheCloudStack.client.beans.ApacheCloudStackUser;
+import br.com.autonomiccs.apacheCloudStack.exceptions.ApacheCloudStackClientRequestRuntimeException;
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementDriver;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
+import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public final class NestedCloudStackOutOfBandManagementDriver extends AdapterBase implements OutOfBandManagementDriver {
+    private static final Logger LOG = Logger.getLogger(NestedCloudStackOutOfBandManagementDriver.class);
+
+    public OutOfBandManagementDriverResponse execute(final OutOfBandManagementDriverCommand cmd) {
+        OutOfBandManagementDriverResponse response = new OutOfBandManagementDriverResponse(null, "Unsupported Command", false);
+
+        if (cmd instanceof OutOfBandManagementDriverPowerCommand) {
+            response = execute((OutOfBandManagementDriverPowerCommand) cmd);
+        } else if (cmd instanceof OutOfBandManagementDriverChangePasswordCommand) {
+            throw new CloudRuntimeException("Change password operation is not supported by the nested-cloudstack out-of-band management driver");
+        }
+
+        return response;
+    }
+
+    protected void ensureOptionExists(final ImmutableMap<OutOfBandManagement.Option, String> options, final OutOfBandManagement.Option option) {
+        if (options != null && option != null && options.containsKey(option) && !Strings.isNullOrEmpty(options.get(option))) {
+            return;
+        }
+        throw new CloudRuntimeException("Invalid out-of-band management configuration detected for the nested-cloudstack driver");
+    }
+
+    protected OutOfBandManagement.PowerState getNestedVMPowerState(final String jsonResponse) {
+        if (Strings.isNullOrEmpty(jsonResponse)) {
+            return OutOfBandManagement.PowerState.Unknown;
+        }
+
+        final ObjectMapper mapper = new ObjectMapper();
+        try {
+            Map<String, Object> listResponse = mapper.readValue(jsonResponse, Map.class);
+            if (listResponse != null && listResponse.containsKey("listvirtualmachinesresponse")
+                    && ((Map<String, Object>) listResponse.get("listvirtualmachinesresponse")).containsKey("virtualmachine")) {
+                Map<String, String> vmResponse = ((Map<String, List<Map<String, String>>>) listResponse.get("listvirtualmachinesresponse")).get("virtualmachine").get(0);
+                if (vmResponse != null && vmResponse.containsKey("state")) {
+                    if("Running".equals(vmResponse.get("state"))) {
+                        return OutOfBandManagement.PowerState.On;
+                    } else if("Stopped".equals(vmResponse.get("state"))) {
+                        return OutOfBandManagement.PowerState.Off;
+                    }
+                }
+            }
+        } catch (IOException e) {
+            LOG.warn("Exception caught while de-serializing and reading state of the nested-cloudstack VM from the response: " + jsonResponse + ", with exception:", e);
+        }
+        return OutOfBandManagement.PowerState.Unknown;
+    }
+
+    private OutOfBandManagementDriverResponse execute(final OutOfBandManagementDriverPowerCommand cmd) {
+        if (cmd == null || cmd.getPowerOperation() == null) {
+            throw new CloudRuntimeException("Invalid out-of-band management power command provided to the nested-cloudstack driver");
+        }
+
+        final ImmutableMap<OutOfBandManagement.Option, String> options = cmd.getOptions();
+        ensureOptionExists(options, OutOfBandManagement.Option.ADDRESS);
+        ensureOptionExists(options, OutOfBandManagement.Option.PORT);
+        ensureOptionExists(options, OutOfBandManagement.Option.USERNAME);
+        ensureOptionExists(options, OutOfBandManagement.Option.PASSWORD);
+
+        final String url = options.get(OutOfBandManagement.Option.ADDRESS);
+        final String vmUuid = options.get(OutOfBandManagement.Option.PORT);
+        final String apiKey = options.get(OutOfBandManagement.Option.USERNAME);
+        final String secretKey = options.get(OutOfBandManagement.Option.PASSWORD);
+
+        final ApacheCloudStackUser apacheCloudStackUser = new ApacheCloudStackUser(secretKey, apiKey);
+        final ApacheCloudStackClient client = new ApacheCloudStackClient(url, apacheCloudStackUser);
+        client.setValidateServerHttpsCertificate(false);
+        client.setShouldRequestsExpire(false);
+        client.setConnectionTimeout((int) cmd.getTimeout().getStandardSeconds());
+
+        String apiName = "listVirtualMachines";
+        switch (cmd.getPowerOperation()) {
+            case ON:
+                apiName = "startVirtualMachine";
+                break;
+            case OFF:
+            case SOFT:
+                apiName = "stopVirtualMachine";
+                break;
+            case CYCLE:
+            case RESET:
+                apiName = "rebootVirtualMachine";
+                break;
+        }
+
+        final ApacheCloudStackRequest apacheCloudStackRequest = new ApacheCloudStackRequest(apiName);
+        apacheCloudStackRequest.addParameter("response", "json");
+        apacheCloudStackRequest.addParameter("forced", "true");
+        apacheCloudStackRequest.addParameter("id", vmUuid);
+
+        final String apiResponse;
+        try {
+            apiResponse = client.executeRequest(apacheCloudStackRequest);
+        } catch (final ApacheCloudStackClientRequestRuntimeException e) {
+            LOG.error("Nested CloudStack oobm plugin failed due to API error: ", e);
+            final OutOfBandManagementDriverResponse failedResponse = new OutOfBandManagementDriverResponse(e.getResponse(), "HTTP error code: " + e.getStatusCode(), false);
+            if (e.getStatusCode() == 401) {
+                failedResponse.setAuthFailure(true);
+            }
+            return failedResponse;
+        }
+
+        final OutOfBandManagementDriverResponse response = new OutOfBandManagementDriverResponse(apiResponse, null, true);
+        if (OutOfBandManagement.PowerOperation.STATUS.equals(cmd.getPowerOperation())) {
+            response.setPowerState(getNestedVMPowerState(apiResponse));
+        }
+        return response;
+    }
+}

plugins/outofbandmanagement-drivers/nested-cloudstack/test/org/apache/cloudstack/outofbandmanagement/driver/nestedcloudstack/NestedCloudStackOutOfBandManagementDriverTest.java

@@ -0,0 +1,75 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.outofbandmanagement.driver.nestedcloudstack;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.collect.ImmutableMap;
+import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.io.IOException;
+
+@RunWith(MockitoJUnitRunner.class)
+public class NestedCloudStackOutOfBandManagementDriverTest {
+    private NestedCloudStackOutOfBandManagementDriver driver = new NestedCloudStackOutOfBandManagementDriver();
+
+    @Test
+    public void testEnsureOptionExists() throws IOException {
+        final ImmutableMap.Builder<OutOfBandManagement.Option, String> builder = ImmutableMap.builder();
+        builder.put(OutOfBandManagement.Option.ADDRESS, "http://some.cloud/client/api");
+        final ImmutableMap<OutOfBandManagement.Option, String> options = builder.build();
+        driver.ensureOptionExists(options, OutOfBandManagement.Option.ADDRESS);
+
+        boolean caughtException = false;
+        try {
+            driver.ensureOptionExists(options, OutOfBandManagement.Option.PORT);
+        } catch (CloudRuntimeException e) {
+            caughtException = true;
+        }
+        Assert.assertTrue(caughtException);
+    }
+
+    @Test
+    public void testIsVMRunningTrue() throws IOException {
+        String json = "{\"listvirtualmachinesresponse\":{\"count\":1,\"virtualmachine\":[{\"id\":\"38fa7380-9543-486a-b083-190ecf726ba4\",\"name\":\"test-vm\",\"displayname\":\"test-vm\",\"account\":\"admin\",\"userid\":\"78ed9ce8-f3ee-11e4-91ab-00012e4fde1c\",\"username\":\"admin\",\"domainid\":\"53601d4b-f3ee-11e4-91ab-00012e4fde1c\",\"domain\":\"ROOT\",\"created\":\"2017-04-04T19:50:56+0200\",\"state\":\"Running\"}]}}";
+        Assert.assertEquals(driver.getNestedVMPowerState(json), OutOfBandManagement.PowerState.On);
+    }
+
+    @Test
+    public void testIsVMRunningFalse() throws IOException {
+        String json = "{\"listvirtualmachinesresponse\":{\"count\":1,\"virtualmachine\":[{\"id\":\"38fa7380-9543-486a-b083-190ecf726ba4\",\"name\":\"test-vm\",\"displayname\":\"test-vm\",\"account\":\"admin\",\"userid\":\"78ed9ce8-f3ee-11e4-91ab-00012e4fde1c\",\"username\":\"admin\",\"domainid\":\"53601d4b-f3ee-11e4-91ab-00012e4fde1c\",\"domain\":\"ROOT\",\"created\":\"2017-04-04T19:50:56+0200\",\"state\":\"Stopped\"}]}}";
+        Assert.assertEquals(driver.getNestedVMPowerState(json), OutOfBandManagement.PowerState.Off);
+    }
+
+    @Test
+    public void testIsVMRunningInvalidJson() throws IOException {
+        String json = "{\"listvirtualmachinesresponse\":{\"count\":1,\"virtualmachine\"83-190ecf726ba4\",\"name";
+        Assert.assertEquals(driver.getNestedVMPowerState(json), OutOfBandManagement.PowerState.Unknown);
+    }
+
+    @Test
+    public void testIsVMRunningEmptyJson() throws IOException {
+        String json = "{}";
+        Assert.assertEquals(driver.getNestedVMPowerState(json), OutOfBandManagement.PowerState.Unknown);
+    }
+}

plugins/pom.xml

@@ -83,6 +83,7 @@
     <module>network-elements/stratosphere-ssp</module>
     <module>network-elements/opendaylight</module>
     <module>outofbandmanagement-drivers/ipmitool</module>
+    <module>outofbandmanagement-drivers/nested-cloudstack</module>
     <module>storage-allocators/random</module>
     <module>user-authenticators/ldap</module>
     <module>user-authenticators/md5</module>

server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java

@@ -138,7 +138,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
                     outOfBandManagementConfig.setAddress(value);
                     break;
                 case PORT:
-                    outOfBandManagementConfig.setPort(Integer.parseInt(value));
+                    outOfBandManagementConfig.setPort(value);
                     break;
                 case USERNAME:
                     outOfBandManagementConfig.setUsername(value);
@@ -166,9 +166,7 @@ public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOf
                     value = outOfBandManagementConfig.getAddress();
                     break;
                 case PORT:
-                    if (outOfBandManagementConfig.getPort() != null) {
-                        value = String.valueOf(outOfBandManagementConfig.getPort());
-                    }
+                    value = outOfBandManagementConfig.getPort();
                     break;
                 case USERNAME:
                     value = outOfBandManagementConfig.getUsername();

server/test/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceTest.java

@@ -101,7 +101,7 @@ public class OutOfBandManagementServiceTest {
         OutOfBandManagement config = new OutOfBandManagementVO(123L);
         config.setAddress("localhost");
         config.setDriver("ipmitool");
-        config.setPort(1234);
+        config.setPort("1234");
         ImmutableMap<OutOfBandManagement.Option, String> options = oobmService.getOptions(config);
         Assert.assertEquals(options.get(OutOfBandManagement.Option.ADDRESS), "localhost");
         Assert.assertEquals(options.get(OutOfBandManagement.Option.DRIVER), "ipmitool");

setup/db/db/schema-41000to41100.sql

@@ -241,3 +241,6 @@ CREATE VIEW `cloud`.`host_view` AS
             left join
         `cloud`.`ha_config` ON ha_config.resource_id=host.id
             and ha_config.resource_type='Host';
+
+-- Out-of-band management driver for nested-cloudstack
+ALTER TABLE `cloud`.`oobm` MODIFY COLUMN port VARCHAR(255);

test/integration/smoke/test_outofbandmanagement_nestedplugin.py

@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import marvin
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.lib.utils import *
+from marvin.lib.base import *
+from marvin.lib.common import *
+from nose.plugins.attrib import attr
+
+from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
+
+import socket
+import sys
+import thread
+import time
+
+
+apiRequests = []
+state = "Running"
+
+
+class MockedCloudStackServer(BaseHTTPRequestHandler):
+    """
+        Mocked ACS Mgmt Server
+    """
+    def do_GET(self):
+        global apiRequests, state
+        command = self.path.split('command=')[1].split('&')[0]
+        if command == 'startVirtualMachine':
+            state = "Running"
+        elif command == 'stopVirtualMachine':
+            state = "Stopped"
+        elif command == 'rebootVirtualMachine':
+            state = "Running"
+
+        apiRequests.append(command)
+
+        self.send_response(200)
+        self.send_header('Content-type','application/json')
+        self.end_headers()
+
+        json = "{\"listvirtualmachinesresponse\":{\"count\":1,\"virtualmachine\":[{\"id\":\"some-uuid\",\"name\":\"test-vm\",\"state\":\"%s\"}]}}" % state
+        self.wfile.write(json)
+
+    def log_message(self, format, *args):
+        return
+
+
+class TestOutOfBandManagement(cloudstackTestCase):
+    """ Test cases for out of band management
+    """
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.hypervisor = self.testClient.getHypervisorInfo()
+        self.dbclient = self.testClient.getDbConnection()
+        self.services = self.testClient.getParsedTestDataConfig()
+        self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__
+
+        self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
+        self.host = None
+        self.server = None
+
+        # use random port for mocked-mgmt server
+        s = socket.socket()
+        s.bind(('', 0))
+        self.serverPort = s.getsockname()[1]
+        s.close()
+
+        self.cleanup = []
+        global state, apiRequests
+        state = "Running"
+        apiRequests = []
+
+
+    def tearDown(self):
+        try:
+            self.dbclient.execute("delete from oobm where driver='nestedcloudstack' and port='some-uuid'")
+            cleanup_resources(self.apiclient, self.cleanup)
+            if self.server:
+                self.server.socket.close()
+            global apiRequests
+            apiRequests = []
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+
+    def getHost(self, hostId=None):
+        if self.host and hostId is None:
+            return self.host
+
+        response = list_hosts(
+                        self.apiclient,
+                        zoneid=self.zone.id,
+                        type='Routing',
+                        id=hostId)
+        if len(response) > 0:
+            self.host = response[0]
+            return self.host
+        raise self.skipTest("No hosts found, skipping out-of-band management test")
+
+
+    def getServerIp(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect((self.mgtSvrDetails["mgtSvrIp"], self.mgtSvrDetails["port"]))
+        return s.getsockname()[0]
+
+
+    def getServerPort(self):
+        return self.serverPort
+
+
+    def getOobmConfigCmd(self):
+        cmd = configureOutOfBandManagement.configureOutOfBandManagementCmd()
+        cmd.driver = 'nestedcloudstack'
+        cmd.address = 'http://%s:%s/client/api' % (self.getServerIp(), self.getServerPort())
+        cmd.port = 'some-uuid'
+        cmd.username = 'admin'
+        cmd.password = 'password'
+        cmd.hostid = self.getHost().id
+        return cmd
+
+
+    def getOobmEnableCmd(self):
+        cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
+        cmd.hostid = self.getHost().id
+        return cmd
+
+
+    def getOobmIssueActionCmd(self):
+        cmd = issueOutOfBandManagementPowerAction.issueOutOfBandManagementPowerActionCmd()
+        cmd.hostid = self.getHost().id
+        cmd.action = 'STATUS'
+        return cmd
+
+
+    def issuePowerActionCmd(self, action):
+        cmd = self.getOobmIssueActionCmd()
+        cmd.action = action
+        return self.apiclient.issueOutOfBandManagementPowerAction(cmd)
+
+
+    def configureAndEnableOobm(self):
+        self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
+        response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd())
+        self.assertEqual(response.enabled, True)
+
+
+    def startMgmtServer(self):
+        def startMgmtServer(tname, server):
+            self.debug("Starting ACS mocked-mgmt server")
+            try:
+                server.serve_forever()
+            except Exception: pass
+        server = HTTPServer(('0.0.0.0', self.getServerPort()), MockedCloudStackServer)
+        thread.start_new_thread(startMgmtServer, ("mocked-mgmt-server", server,))
+        self.server = server
+
+
+    def configureAndStartMgmtServer(self):
+        """
+            Configure mocked-mgmt server and enable out-of-band management for host
+        """
+        self.configureAndEnableOobm()
+        self.startMgmtServer()
+
+
+    def assertIssueCommandState(self, command, expected):
+        """
+            Asserts power action result for a given power command
+        """
+
+        if command != 'STATUS':
+            self.issuePowerActionCmd(command)
+        response = self.issuePowerActionCmd('STATUS')
+        self.assertEqual(response.powerstate, expected)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_status(self):
+        """
+            Tests out-of-band management issue power action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('STATUS', 'On')
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_on(self):
+        """
+            Tests out-of-band management issue power on action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('ON', 'On')
+        global apiRequests
+        self.assertTrue('startVirtualMachine' in apiRequests)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_off(self):
+        """
+            Tests out-of-band management issue power off action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('OFF', 'Off')
+        global apiRequests
+        self.assertTrue('stopVirtualMachine' in apiRequests)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_cycle(self):
+        """
+            Tests out-of-band management issue power cycle action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('CYCLE', 'On')
+        global apiRequests
+        self.assertTrue('rebootVirtualMachine' in apiRequests)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_reset(self):
+        """
+            Tests out-of-band management issue power reset action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('RESET', 'On')
+        global apiRequests
+        self.assertTrue('rebootVirtualMachine' in apiRequests)
+
+
+    @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_oobm_issue_power_soft(self):
+        """
+            Tests out-of-band management issue power soft action
+        """
+        self.configureAndStartMgmtServer()
+        self.assertIssueCommandState('SOFT', 'Off')
+        global apiRequests
+        self.assertTrue('stopVirtualMachine' in apiRequests)

ui/scripts/system.js

@@ -16786,6 +16786,10 @@
                                                     id: 'ipmitool',
                                                     description: 'ipmitool - ipmitool based shell driver'
                                                 });
+                                                items.push({
+                                                    id: 'nestedcloudstack',
+                                                    description: 'nested-cloudstack - controls host that is a VM in a parent cloudstack (testing purposes only)'
+                                                });
                                                 args.response.success({
                                                     data: items
                                                 });