CLOUDSTACK-6090: Virtual Router Service Failure Alerting

Signed-off-by: Koushik Das <koushik@apache.org>

core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api;
+
+
+import com.cloud.agent.api.routing.GetRouterAlertsCommand;
+
+public class GetRouterAlertsAnswer extends Answer {
+
+    String routerName;
+    String[] alerts;
+    String timeStamp;
+
+    protected GetRouterAlertsAnswer() {
+    }
+
+    public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, String alerts[], String timeStamp) {
+        super(cmd, true, null);
+        this.alerts = alerts;
+        this.timeStamp = timeStamp;
+    }
+
+
+    public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, Exception ex) {
+        super(cmd, ex);
+    }
+
+    public void setAlerts(String[] alerts) {
+        this.alerts = alerts;
+    }
+
+    public String[] getAlerts() {
+        return alerts;
+    }
+
+    public void setTimeStamp(String timeStamp) {
+        this.timeStamp = timeStamp;
+    }
+
+    public String getTimeStamp() {
+        return timeStamp;
+    }
+
+    public String getRouterName() {
+        return routerName;
+    }
+}

core/src/com/cloud/agent/api/routing/GetRouterAlertsCommand.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api.routing;
+
+
+public class GetRouterAlertsCommand  extends NetworkElementCommand {
+
+    private String previousAlertTimeStamp;
+
+    protected GetRouterAlertsCommand() {
+
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public GetRouterAlertsCommand(String timeStamp) {
+        this.previousAlertTimeStamp = timeStamp;
+    }
+
+    public String getPreviousAlertTimeStamp() {
+        return previousAlertTimeStamp;
+    }
+
+    @Override
+    public boolean isQuery() {
+        return true;
+    }
+}

core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java

@@ -32,6 +32,8 @@ import com.cloud.agent.api.routing.DeleteIpAliasCommand;
 import com.cloud.agent.api.routing.DhcpEntryCommand;
 import com.cloud.agent.api.routing.DnsMasqConfigCommand;
 import com.cloud.agent.api.routing.GroupAnswer;
+import com.cloud.agent.api.routing.GetRouterAlertsCommand;
+import com.cloud.agent.api.GetRouterAlertsAnswer;
 import com.cloud.agent.api.routing.IpAliasTO;
 import com.cloud.agent.api.routing.IpAssocCommand;
 import com.cloud.agent.api.routing.IpAssocVpcCommand;
@@ -102,6 +104,7 @@ public class VirtualRoutingResource {
         protected static final String IPASSOC = "ipassoc.sh";
         protected static final String LB = "loadbalancer.sh";
         protected static final String MONITOR_SERVICE = "monitor_service.sh";
+        protected static final String ROUTER_ALERTS = "getRouterAlerts.sh";
         protected static final String PASSWORD = "savepassword.sh";
         protected static final String RVR_CHECK = "checkrouter.sh";
         protected static final String RVR_BUMPUP_PRI = "bumpup_priority.sh";
@@ -275,7 +278,9 @@ public class VirtualRoutingResource {
         } else if (cmd instanceof GetDomRVersionCmd) {
             return execute((GetDomRVersionCmd)cmd);
         } else if (cmd instanceof CheckS2SVpnConnectionsCommand) {
-            return execute((CheckS2SVpnConnectionsCommand)cmd);
+            return execute((CheckS2SVpnConnectionsCommand) cmd);
+        } else if (cmd instanceof GetRouterAlertsCommand) {
+            return execute((GetRouterAlertsCommand)cmd);
         } else {
             s_logger.error("Unknown query command in VirtualRoutingResource!");
             return Answer.createUnsupportedCommandAnswer(cmd);
@@ -642,6 +647,29 @@ public class VirtualRoutingResource {
         return new CheckS2SVpnConnectionsAnswer(cmd, result.isSuccess(), result.getDetails());
     }
 
+    private GetRouterAlertsAnswer execute(GetRouterAlertsCommand cmd) {
+
+        String args = null;
+        String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
+        if (cmd.getPreviousAlertTimeStamp() != null) {
+            args = cmd.getPreviousAlertTimeStamp();
+        }
+
+        ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_ALERTS, args);
+        String alerts[] = null;
+        String lastAlertTimestamp = null;
+        // CallHostPlugin results "success" when there are no alerts on virtual router
+        if (result.isSuccess()) {
+            if (!result.getDetails().isEmpty() && !result.getDetails().equals("No Alerts")) {
+                alerts = result.getDetails().split("\\\\n");
+                String[] lastAlert = alerts[alerts.length - 1].split(" ");
+                lastAlertTimestamp = lastAlert[0] + " " + lastAlert[1];
+            }
+        }
+
+        return new GetRouterAlertsAnswer(cmd, alerts, lastAlertTimestamp);
+    }
+
     protected Answer execute(CheckRouterCommand cmd) {
         final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RVR_CHECK, null);
         if (!result.isSuccess()) {
@@ -732,6 +760,21 @@ public class VirtualRoutingResource {
         return cfg;
     }
 
+    protected List<ConfigItem> generateConfig(GetRouterAlertsCommand cmd) {
+        LinkedList<ConfigItem> cfg = new LinkedList<>();
+
+        String args = null;
+        String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
+        if (cmd.getPreviousAlertTimeStamp() != null) {
+            args = "getRouterAlerts.sh " + routerIp + " " + cmd.getPreviousAlertTimeStamp();
+        } else {
+            args = "getRouterAlerts.sh " + routerIp;
+        }
+
+        cfg.add(new ConfigItem(VRScripts.ROUTER_ALERTS, args));
+        return cfg;
+    }
+
     protected List<ConfigItem> generateConfig(SetupGuestNetworkCommand cmd) {
         LinkedList<ConfigItem> cfg = new LinkedList<>();
 

engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -157,6 +157,7 @@
   <bean id="firewallRulesCidrsDaoImpl" class="com.cloud.network.dao.FirewallRulesCidrsDaoImpl" />
   <bean id="firewallRulesDaoImpl" class="com.cloud.network.dao.FirewallRulesDaoImpl" />
   <bean id="MonitoringServiceDaoImpl" class="com.cloud.network.dao.MonitoringServiceDaoImpl" />
+  <bean id="OpRouterMonitorServiceDaoImpl" class="com.cloud.network.dao.OpRouterMonitorServiceDaoImpl" />
   <bean id="globalLoadBalancerDaoImpl" class="org.apache.cloudstack.region.gslb.GlobalLoadBalancerDaoImpl" />
   <bean id="globalLoadBalancerLbRuleMapDaoImpl" class="org.apache.cloudstack.region.gslb.GlobalLoadBalancerLbRuleMapDaoImpl" />
   <bean id="guestOSCategoryDaoImpl" class="com.cloud.storage.dao.GuestOSCategoryDaoImpl" />

engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDao.java

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.network.dao;
+
+
+import com.cloud.utils.db.GenericDao;
+
+public interface OpRouterMonitorServiceDao extends GenericDao<OpRouterMonitorServiceVO, Long> {
+
+}

engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.network.dao;
+
+import com.cloud.utils.db.GenericDaoBase;
+import org.springframework.stereotype.Component;
+
+import javax.ejb.Local;
+
+@Component
+@Local(value=OpRouterMonitorServiceDao.class)
+public class OpRouterMonitorServiceDaoImpl extends GenericDaoBase<OpRouterMonitorServiceVO, Long> implements OpRouterMonitorServiceDao  {
+}

engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceVO.java

@@ -0,0 +1,52 @@
+package com.cloud.network.dao;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+import javax.persistence.Entity;
+import javax.persistence.Table;
+import javax.persistence.Id;
+import javax.persistence.Column;
+
+
+@Entity
+@Table(name = "op_router_monitoring_services")
+public class OpRouterMonitorServiceVO implements InternalIdentity {
+
+    @Id
+    @Column(name="vm_id")
+    Long id;
+
+    @Column(name="router_name")
+    private String name;
+
+    @Column(name="last_alert_timestamp")
+    private String lastAlertTimestamp;
+
+
+    public OpRouterMonitorServiceVO() {}
+
+    public OpRouterMonitorServiceVO(long vmId, String name, String lastAlertTimestamp) {
+        this.id = vmId;
+        this.name = name;
+        this.lastAlertTimestamp = lastAlertTimestamp;
+    }
+
+
+    @Override
+    public long getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getLastAlertTimestamp() {
+        return lastAlertTimestamp;
+    }
+
+    public void setLastAlertTimestamp (String timestamp) {
+        this.lastAlertTimestamp = timestamp;
+    }
+
+}

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -55,6 +55,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager;
 import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
+import org.apache.cloudstack.alert.AlertService.AlertType;
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
 import com.cloud.agent.api.AgentControlAnswer;
@@ -73,6 +74,8 @@ import com.cloud.agent.api.NetworkUsageAnswer;
 import com.cloud.agent.api.NetworkUsageCommand;
 import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.agent.api.StartupCommand;
+import com.cloud.agent.api.routing.GetRouterAlertsCommand;
+import com.cloud.agent.api.GetRouterAlertsAnswer;
 import com.cloud.agent.api.check.CheckSshAnswer;
 import com.cloud.agent.api.check.CheckSshCommand;
 import com.cloud.agent.api.routing.AggregationControlCommand;
@@ -179,6 +182,7 @@ import com.cloud.network.dao.MonitoringServiceDao;
 import com.cloud.network.dao.MonitoringServiceVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
+import com.cloud.network.dao.OpRouterMonitorServiceDao;
 import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
 import com.cloud.network.dao.RemoteAccessVpnDao;
 import com.cloud.network.dao.Site2SiteCustomerGatewayDao;
@@ -188,6 +192,7 @@ import com.cloud.network.dao.Site2SiteVpnGatewayDao;
 import com.cloud.network.dao.UserIpv6AddressDao;
 import com.cloud.network.dao.VirtualRouterProviderDao;
 import com.cloud.network.dao.VpnUserDao;
+import com.cloud.network.dao.OpRouterMonitorServiceVO;
 import com.cloud.network.lb.LoadBalancingRule;
 import com.cloud.network.lb.LoadBalancingRule.LbDestination;
 import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy;
@@ -394,6 +399,8 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
     AsyncJobManager _asyncMgr;
     @Inject
     protected ApiAsyncJobDispatcher _asyncDispatcher;
+    @Inject
+    OpRouterMonitorServiceDao _opRouterMonitorServiceDao;
 
     int _routerRamSize;
     int _routerCpuMHz;
@@ -1348,6 +1355,8 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
 
                 updateSite2SiteVpnConnectionState(routers);
 
+                getRouterAlerts();
+
                 final List<NetworkVO> networks = _networkDao.listRedundantNetworks();
                 s_logger.debug("Found " + networks.size() + " networks to update RvR status. ");
                 for (final NetworkVO network : networks) {
@@ -1362,6 +1371,64 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
         }
     }
 
+    private void getRouterAlerts() {
+        try{
+            List<DomainRouterVO> routersInIsolatedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId);
+            List<DomainRouterVO> routersInSharedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Shared, mgmtSrvrId);
+
+            List<DomainRouterVO> routers = new ArrayList<DomainRouterVO>();
+            routers.addAll(routersInIsolatedNetwork);
+            routers.addAll(routersInSharedNetwork);
+            s_logger.debug("Found " + routers.size() + " running routers. ");
+
+            for (final DomainRouterVO router : routers) {
+                if (router.getVpcId() != null) {
+                    continue;
+                }
+                String privateIP = router.getPrivateIpAddress();
+
+                if (privateIP != null) {
+                    OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId());
+
+                    GetRouterAlertsCommand command = null;
+                    if (opRouterMonitorServiceVO == null) {
+                        command = new GetRouterAlertsCommand(null);
+                    } else {
+                        command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp());
+                    }
+
+                    command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress());
+                    command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
+
+                    GetRouterAlertsAnswer answer = null;
+                    try {
+                        answer = (GetRouterAlertsAnswer) _agentMgr.easySend(router.getHostId(), command);
+                        String alerts[] = answer.getAlerts();
+                        if (alerts != null ) {
+                            for (String alert: alerts) {
+                                _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), "Monitoring Service on VR " + router.getInstanceName(), alert);
+                            }
+                            String lastAlertTimeStamp = answer.getTimeStamp();
+                            if (opRouterMonitorServiceVO == null) {
+                                opRouterMonitorServiceVO = new OpRouterMonitorServiceVO(router.getId(), router.getHostName(), lastAlertTimeStamp);
+                                _opRouterMonitorServiceDao.persist(opRouterMonitorServiceVO);
+                            } else {
+                                opRouterMonitorServiceVO.setLastAlertTimestamp(lastAlertTimeStamp);
+                                _opRouterMonitorServiceDao.update(opRouterMonitorServiceVO.getId(), opRouterMonitorServiceVO);
+                            }
+                        }
+                    } catch (Exception e) {
+                        s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e);
+                        continue;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            s_logger.warn("Error while collecting alerts from router", e);
+        }
+    }
+
+
     private final static int DEFAULT_PRIORITY = 100;
     private final static int DEFAULT_DELTA = 2;
 

setup/db/db/schema-430to440.sql

@@ -657,6 +657,14 @@ ALTER TABLE `cloud`.`s2s_vpn_gateway` ADD COLUMN `display` tinyint(1) NOT NULL D
 INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (225, UUID(), 9, 'FreeBSD 10 (32-bit)');
 INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (226, UUID(), 9, 'FreeBSD 10 (64-bit)');
 
+CREATE TABLE `cloud`.`op_router_monitoring_services` (
+  `vm_id` bigint unsigned UNIQUE NOT NULL COMMENT 'Primary Key',
+  `router_name` varchar(255) NOT NULL COMMENT 'Name of the Virtual Router',
+  `last_alert_timestamp` varchar(255) NOT NULL COMMENT 'Timestamp of the last alert received from Virtual Router',
+   PRIMARY KEY (`vm_id`),
+   CONSTRAINT `fk_virtual_router__id` FOREIGN KEY `fk_virtual_router__id` (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE
+) ENGINE = InnoDB DEFAULT CHARSET=utf8
+
 ALTER TABLE `cloud`.`event` ADD COLUMN `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user';
 
 DROP VIEW IF EXISTS `cloud`.`event_view`;

systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# getRouterAlerts.sh  --- Send the alerts from routerServiceMonitor.log to Management Server
+
+source /root/func.sh
+
+lock="biglock"
+locked=$(getLockFile $lock)
+if [ "$locked" != "1" ]
+then
+    exit 1
+fi
+
+#set -x
+
+filename=/var/log/routerServiceMonitor.log #Monitor service log file
+if [ -n "$1" -a -n "$2" ]
+then
+        reqdateval=$(date -d $1 +"%Y%m%d");
+        reqtimeval=$(date -d $2 +"%H%M%S");
+else
+        reqdateval=0
+        reqtimeval=0
+fi
+if [ -f $filename ]
+then
+        while read line
+        do
+        if [ -n "$line" ]; then
+            dateval=`echo $line |awk '{print $1}'`
+            timeval=`echo $line |awk '{print $2}'`
+
+            todate=$(date -d "$dateval" +"%Y%m%d") > /dev/null
+            totime=$(date -d "$timeval" +"%H%M%S") > /dev/null
+            if [ "$todate" -gt "$reqdateval" ] > /dev/null
+            then
+                if [ -n "$alerts" ]; then  alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile
+                elif [ "$todate" -eq "$reqdateval" ] > /dev/null
+                then
+                    if [ "$totime" -gt "$reqtimeval" ] > /dev/null
+                    then
+                        if [ -n "$alerts" ]; then  alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile
+                    fi
+                fi
+            fi
+        done < $filename
+fi
+if [ -n "$alerts" ]; then
+       echo $alerts
+else
+       echo "No Alerts"
+fi
+
+unlock_exit 0 $lock $locked

systemvm/patches/debian/config/root/monitorServices.py

@@ -25,6 +25,7 @@ from subprocess import *
 from os import path
 import time
 import os
+import logging
 
 class StatusCodes:
     SUCCESS      = 0
@@ -92,6 +93,8 @@ def raisealert(severity, msg, process_name=None):
     else:
         log = '['+severity+']' + " " + msg +"\n"
 
+    logging.basicConfig(level=logging.INFO,filename='/var/log/routerServiceMonitor.log',format='%(asctime)s %(message)s')
+    logging.info(log)
     msg = 'logger -t monit '+ log
     pout = Popen(msg, shell=True, stdout=PIPE)
 

test/integration/smoke/test_VirtualRouter_alerts.py

@@ -0,0 +1,244 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" P1 tests for alert receiving from VR on service failure in VR
+"""
+#Import Local Modules
+import marvin
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.integration.lib.utils import *
+from marvin.integration.lib.base import *
+from marvin.integration.lib.common import *
+from nose.plugins.attrib import attr
+import time
+
+
+_multiprocess_shared_ = True
+
+class Services:
+    """Test VM Life Cycle Services
+    """
+
+    def __init__(self):
+        self.services = {
+
+            "account": {
+                "email": "test@test.com",
+                "firstname": "Test",
+                "lastname": "User",
+                "username": "test",
+                # Random characters are appended in create account to
+                # ensure unique username generated each time
+                "password": "password",
+                },
+            "small":
+            # Create a small virtual machine instance with disk offering
+                {
+                    "displayname": "testserver",
+                    "username": "root", # VM creds for SSH
+                    "password": "password",
+                    "ssh_port": 22,
+                    "hypervisor": 'XenServer',
+                    "privateport": 22,
+                    "publicport": 22,
+                    "protocol": 'TCP',
+                    },
+            "service_offerings":
+                {
+                    "small":
+                        {
+                            # Small service offering ID to for change VM
+                            # service offering from medium to small
+                            "name": "SmallInstance",
+                            "displaytext": "SmallInstance",
+                            "cpunumber": 1,
+                            "cpuspeed": 100,
+                            "memory": 256,
+                            },
+                    "big":
+                        {
+                            # Big service offering ID to for change VM
+                            "name": "BigInstance",
+                            "displaytext": "BigInstance",
+                            "cpunumber": 1,
+                            "cpuspeed": 100,
+                            "memory": 512,
+                            }
+                },
+            #Change this
+            "template": {
+                "displaytext": "xs",
+                "name": "xs",
+                "passwordenabled": False,
+                },
+            "sleep": 60,
+            "timeout": 10,
+            #Migrate VM to hostid
+            "ostype": 'CentOS 5.3 (64-bit)',
+            # CentOS 5.3 (64-bit)
+        }
+
+
+class TestVRServiceFailureAlerting(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.api_client = super(TestVRServiceFailureAlerting, cls).getClsTestClient().getApiClient()
+        cls.services = Services().services
+
+        # Get Zone, Domain and templates
+        domain = get_domain(cls.api_client, cls.services)
+        cls.zone = get_zone(cls.api_client, cls.services)
+        cls.services['mode'] = cls.zone.networktype
+
+        template = get_template(
+            cls.api_client,
+            cls.zone.id,
+            cls.services["ostype"]
+        )
+        # Set Zones and disk offerings ??
+        cls.services["small"]["zoneid"] = cls.zone.id
+        cls.services["small"]["template"] = template.id
+
+        # Create account, service offerings, vm.
+        cls.account = Account.create(
+            cls.api_client,
+            cls.services["account"],
+            domainid=domain.id
+        )
+
+        cls.small_offering = ServiceOffering.create(
+            cls.api_client,
+            cls.services["service_offerings"]["small"]
+        )
+
+        #create a virtual machine
+        cls.virtual_machine = VirtualMachine.create(
+            cls.api_client,
+            cls.services["small"],
+            accountid=cls.account.name,
+            domainid=cls.account.domainid,
+            serviceofferingid=cls.small_offering.id,
+            mode=cls.services["mode"]
+        )
+        cls._cleanup = [
+            cls.small_offering,
+            cls.account
+        ]
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.api_client = super(TestVRServiceFailureAlerting, cls).getClsTestClient().getApiClient()
+        cleanup_resources(cls.api_client, cls._cleanup)
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.cleanup = []
+
+    def tearDown(self):
+        #Clean up, terminate the created ISOs
+        cleanup_resources(self.apiclient, self.cleanup)
+        return
+
+    @attr(hypervisor="xenserver")
+    @attr(tags=["advanced", "basic"])
+    def test_01_VRServiceFailureAlerting(self):
+
+
+        if self.zone.networktype == "Basic":
+            list_router_response = list_routers(
+                self.apiclient,
+                listall="true"
+            )
+        else:
+            list_router_response = list_routers(
+                self.apiclient,
+                account=self.account.name,
+                domainid=self.account.domainid
+            )
+        self.assertEqual(
+            isinstance(list_router_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+        router = list_router_response[0]
+
+        hosts = list_hosts(
+            self.apiclient,
+            zoneid=router.zoneid,
+            type='Routing',
+            state='Up',
+            id=router.hostid
+        )
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check list host returns a valid list"
+        )
+        host = hosts[0]
+
+        self.debug("Router ID: %s, state: %s" % (router.id, router.state))
+
+        self.assertEqual(
+        router.state,
+        'Running',
+        "Check list router response for router state"
+        )
+
+        alertSubject = "Monitoring Service on VR " + router.name
+
+        if self.apiclient.hypervisor.lower() == 'vmware':
+            result = get_process_status(
+                        self.apiclient.connection.mgtSvr,
+                        22,
+                        self.apiclient.connection.user,
+                        self.apiclient.connection.passwd,
+                        router.linklocalip,
+                        "service dnsmasq status",
+                        hypervisor=self.apiclient.hypervisor
+                        )
+        else:
+            try:
+                host.user, host.passwd = get_host_credentials(self.config, host.ipaddress)
+                result = get_process_status(
+                            host.ipaddress,
+                            22,
+                            host.user,
+                            host.passwd,
+                            router.linklocalip,
+                            "service apache2 stop"
+                            )
+            except KeyError:
+                self.skipTest("Marvin configuration has no host credentials to check router services")
+
+        res = str(result)
+        self.debug("apache process status: %s" % res)
+
+        time.sleep(300) #wait for 5 minutes meanwhile monitor service on VR starts the apache service
+
+        qresultset = self.dbclient.execute(
+            "select id from alert where subject = '%s' ORDER BY id DESC LIMIT 1;" \
+            % str(alertSubject)
+        )
+        self.assertNotEqual(
+            len(qresultset),
+            0,
+            "Check DB Query result set"
+        )
+
+        return