From f7337527cf9dbb35ec4c60621d4b37173635660c Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 14 Mar 2014 15:00:38 +0530 Subject: [PATCH] CLOUDSTACK-6090: Virtual Router Service Failure Alerting Signed-off-by: Koushik Das --- .../agent/api/GetRouterAlertsAnswer.java | 62 +++++ .../api/routing/GetRouterAlertsCommand.java | 46 ++++ .../VirtualRoutingResource.java | 45 +++- ...spring-engine-schema-core-daos-context.xml | 1 + .../dao/OpRouterMonitorServiceDao.java | 25 ++ .../dao/OpRouterMonitorServiceDaoImpl.java | 28 ++ .../network/dao/OpRouterMonitorServiceVO.java | 52 ++++ .../VirtualNetworkApplianceManagerImpl.java | 67 +++++ setup/db/db/schema-430to440.sql | 8 + .../config/opt/cloud/bin/getRouterAlerts.sh | 70 +++++ .../debian/config/root/monitorServices.py | 3 + .../smoke/test_VirtualRouter_alerts.py | 244 ++++++++++++++++++ 12 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java create mode 100644 core/src/com/cloud/agent/api/routing/GetRouterAlertsCommand.java create mode 100644 engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDao.java create mode 100644 engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java create mode 100644 engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceVO.java create mode 100644 systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh create mode 100644 test/integration/smoke/test_VirtualRouter_alerts.py diff --git a/core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java b/core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java new file mode 100644 index 00000000000..06a7a7a84eb --- /dev/null +++ b/core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + + +import com.cloud.agent.api.routing.GetRouterAlertsCommand; + +public class GetRouterAlertsAnswer extends Answer { + + String routerName; + String[] alerts; + String timeStamp; + + protected GetRouterAlertsAnswer() { + } + + public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, String alerts[], String timeStamp) { + super(cmd, true, null); + this.alerts = alerts; + this.timeStamp = timeStamp; + } + + + public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, Exception ex) { + super(cmd, ex); + } + + public void setAlerts(String[] alerts) { + this.alerts = alerts; + } + + public String[] getAlerts() { + return alerts; + } + + public void setTimeStamp(String timeStamp) { + this.timeStamp = timeStamp; + } + + public String getTimeStamp() { + return timeStamp; + } + + public String getRouterName() { + return routerName; + } +} diff --git a/core/src/com/cloud/agent/api/routing/GetRouterAlertsCommand.java b/core/src/com/cloud/agent/api/routing/GetRouterAlertsCommand.java new file mode 100644 index 00000000000..a6769efa4ae --- /dev/null +++ b/core/src/com/cloud/agent/api/routing/GetRouterAlertsCommand.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + + +public class GetRouterAlertsCommand extends NetworkElementCommand { + + private String previousAlertTimeStamp; + + protected GetRouterAlertsCommand() { + + } + + @Override + public boolean executeInSequence() { + return false; + } + + public GetRouterAlertsCommand(String timeStamp) { + this.previousAlertTimeStamp = timeStamp; + } + + public String getPreviousAlertTimeStamp() { + return previousAlertTimeStamp; + } + + @Override + public boolean isQuery() { + return true; + } +} diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 3712abad7d8..e13fbf651cb 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -32,6 +32,8 @@ import com.cloud.agent.api.routing.DeleteIpAliasCommand; import com.cloud.agent.api.routing.DhcpEntryCommand; import com.cloud.agent.api.routing.DnsMasqConfigCommand; import com.cloud.agent.api.routing.GroupAnswer; +import com.cloud.agent.api.routing.GetRouterAlertsCommand; +import com.cloud.agent.api.GetRouterAlertsAnswer; import com.cloud.agent.api.routing.IpAliasTO; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.IpAssocVpcCommand; @@ -102,6 +104,7 @@ public class VirtualRoutingResource { protected static final String IPASSOC = "ipassoc.sh"; protected static final String LB = "loadbalancer.sh"; protected static final String MONITOR_SERVICE = "monitor_service.sh"; + protected static final String ROUTER_ALERTS = "getRouterAlerts.sh"; protected static final String PASSWORD = "savepassword.sh"; protected static final String RVR_CHECK = "checkrouter.sh"; protected static final String RVR_BUMPUP_PRI = "bumpup_priority.sh"; @@ -275,7 +278,9 @@ public class VirtualRoutingResource { } else if (cmd instanceof GetDomRVersionCmd) { return execute((GetDomRVersionCmd)cmd); } else if (cmd instanceof CheckS2SVpnConnectionsCommand) { - return execute((CheckS2SVpnConnectionsCommand)cmd); + return execute((CheckS2SVpnConnectionsCommand) cmd); + } else if (cmd instanceof GetRouterAlertsCommand) { + return execute((GetRouterAlertsCommand)cmd); } else { s_logger.error("Unknown query command in VirtualRoutingResource!"); return Answer.createUnsupportedCommandAnswer(cmd); @@ -642,6 +647,29 @@ public class VirtualRoutingResource { return new CheckS2SVpnConnectionsAnswer(cmd, result.isSuccess(), result.getDetails()); } + private GetRouterAlertsAnswer execute(GetRouterAlertsCommand cmd) { + + String args = null; + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + if (cmd.getPreviousAlertTimeStamp() != null) { + args = cmd.getPreviousAlertTimeStamp(); + } + + ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_ALERTS, args); + String alerts[] = null; + String lastAlertTimestamp = null; + // CallHostPlugin results "success" when there are no alerts on virtual router + if (result.isSuccess()) { + if (!result.getDetails().isEmpty() && !result.getDetails().equals("No Alerts")) { + alerts = result.getDetails().split("\\\\n"); + String[] lastAlert = alerts[alerts.length - 1].split(" "); + lastAlertTimestamp = lastAlert[0] + " " + lastAlert[1]; + } + } + + return new GetRouterAlertsAnswer(cmd, alerts, lastAlertTimestamp); + } + protected Answer execute(CheckRouterCommand cmd) { final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RVR_CHECK, null); if (!result.isSuccess()) { @@ -732,6 +760,21 @@ public class VirtualRoutingResource { return cfg; } + protected List generateConfig(GetRouterAlertsCommand cmd) { + LinkedList cfg = new LinkedList<>(); + + String args = null; + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + if (cmd.getPreviousAlertTimeStamp() != null) { + args = "getRouterAlerts.sh " + routerIp + " " + cmd.getPreviousAlertTimeStamp(); + } else { + args = "getRouterAlerts.sh " + routerIp; + } + + cfg.add(new ConfigItem(VRScripts.ROUTER_ALERTS, args)); + return cfg; + } + protected List generateConfig(SetupGuestNetworkCommand cmd) { LinkedList cfg = new LinkedList<>(); diff --git a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 765d86c34ad..489b37d04b7 100644 --- a/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -157,6 +157,7 @@ + diff --git a/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDao.java b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDao.java new file mode 100644 index 00000000000..ebc0f1af227 --- /dev/null +++ b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDao.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.network.dao; + + +import com.cloud.utils.db.GenericDao; + +public interface OpRouterMonitorServiceDao extends GenericDao { + +} diff --git a/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java new file mode 100644 index 00000000000..b92512e6d91 --- /dev/null +++ b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.network.dao; + +import com.cloud.utils.db.GenericDaoBase; +import org.springframework.stereotype.Component; + +import javax.ejb.Local; + +@Component +@Local(value=OpRouterMonitorServiceDao.class) +public class OpRouterMonitorServiceDaoImpl extends GenericDaoBase implements OpRouterMonitorServiceDao { +} diff --git a/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceVO.java b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceVO.java new file mode 100644 index 00000000000..c70bba357c1 --- /dev/null +++ b/engine/schema/src/com/cloud/network/dao/OpRouterMonitorServiceVO.java @@ -0,0 +1,52 @@ +package com.cloud.network.dao; + +import org.apache.cloudstack.api.InternalIdentity; + +import javax.persistence.Entity; +import javax.persistence.Table; +import javax.persistence.Id; +import javax.persistence.Column; + + +@Entity +@Table(name = "op_router_monitoring_services") +public class OpRouterMonitorServiceVO implements InternalIdentity { + + @Id + @Column(name="vm_id") + Long id; + + @Column(name="router_name") + private String name; + + @Column(name="last_alert_timestamp") + private String lastAlertTimestamp; + + + public OpRouterMonitorServiceVO() {} + + public OpRouterMonitorServiceVO(long vmId, String name, String lastAlertTimestamp) { + this.id = vmId; + this.name = name; + this.lastAlertTimestamp = lastAlertTimestamp; + } + + + @Override + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getLastAlertTimestamp() { + return lastAlertTimestamp; + } + + public void setLastAlertTimestamp (String timestamp) { + this.lastAlertTimestamp = timestamp; + } + +} diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index eeab91d81ab..c692491a34b 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -55,6 +55,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.alert.AlertService.AlertType; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -73,6 +74,8 @@ import com.cloud.agent.api.NetworkUsageAnswer; import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.PvlanSetupCommand; import com.cloud.agent.api.StartupCommand; +import com.cloud.agent.api.routing.GetRouterAlertsCommand; +import com.cloud.agent.api.GetRouterAlertsAnswer; import com.cloud.agent.api.check.CheckSshAnswer; import com.cloud.agent.api.check.CheckSshCommand; import com.cloud.agent.api.routing.AggregationControlCommand; @@ -179,6 +182,7 @@ import com.cloud.network.dao.MonitoringServiceDao; import com.cloud.network.dao.MonitoringServiceVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.OpRouterMonitorServiceDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.RemoteAccessVpnDao; import com.cloud.network.dao.Site2SiteCustomerGatewayDao; @@ -188,6 +192,7 @@ import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.UserIpv6AddressDao; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.dao.VpnUserDao; +import com.cloud.network.dao.OpRouterMonitorServiceVO; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; @@ -394,6 +399,8 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V AsyncJobManager _asyncMgr; @Inject protected ApiAsyncJobDispatcher _asyncDispatcher; + @Inject + OpRouterMonitorServiceDao _opRouterMonitorServiceDao; int _routerRamSize; int _routerCpuMHz; @@ -1348,6 +1355,8 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V updateSite2SiteVpnConnectionState(routers); + getRouterAlerts(); + final List networks = _networkDao.listRedundantNetworks(); s_logger.debug("Found " + networks.size() + " networks to update RvR status. "); for (final NetworkVO network : networks) { @@ -1362,6 +1371,64 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V } } + private void getRouterAlerts() { + try{ + List routersInIsolatedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); + List routersInSharedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Shared, mgmtSrvrId); + + List routers = new ArrayList(); + routers.addAll(routersInIsolatedNetwork); + routers.addAll(routersInSharedNetwork); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (final DomainRouterVO router : routers) { + if (router.getVpcId() != null) { + continue; + } + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId()); + + GetRouterAlertsCommand command = null; + if (opRouterMonitorServiceVO == null) { + command = new GetRouterAlertsCommand(null); + } else { + command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp()); + } + + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress()); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + + GetRouterAlertsAnswer answer = null; + try { + answer = (GetRouterAlertsAnswer) _agentMgr.easySend(router.getHostId(), command); + String alerts[] = answer.getAlerts(); + if (alerts != null ) { + for (String alert: alerts) { + _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), "Monitoring Service on VR " + router.getInstanceName(), alert); + } + String lastAlertTimeStamp = answer.getTimeStamp(); + if (opRouterMonitorServiceVO == null) { + opRouterMonitorServiceVO = new OpRouterMonitorServiceVO(router.getId(), router.getHostName(), lastAlertTimeStamp); + _opRouterMonitorServiceDao.persist(opRouterMonitorServiceVO); + } else { + opRouterMonitorServiceVO.setLastAlertTimestamp(lastAlertTimeStamp); + _opRouterMonitorServiceDao.update(opRouterMonitorServiceVO.getId(), opRouterMonitorServiceVO); + } + } + } catch (Exception e) { + s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); + continue; + } + } + } + } catch (Exception e) { + s_logger.warn("Error while collecting alerts from router", e); + } + } + + private final static int DEFAULT_PRIORITY = 100; private final static int DEFAULT_DELTA = 2; diff --git a/setup/db/db/schema-430to440.sql b/setup/db/db/schema-430to440.sql index 056c5f84f53..9d41fe92071 100644 --- a/setup/db/db/schema-430to440.sql +++ b/setup/db/db/schema-430to440.sql @@ -657,6 +657,14 @@ ALTER TABLE `cloud`.`s2s_vpn_gateway` ADD COLUMN `display` tinyint(1) NOT NULL D INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (225, UUID(), 9, 'FreeBSD 10 (32-bit)'); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name) VALUES (226, UUID(), 9, 'FreeBSD 10 (64-bit)'); +CREATE TABLE `cloud`.`op_router_monitoring_services` ( + `vm_id` bigint unsigned UNIQUE NOT NULL COMMENT 'Primary Key', + `router_name` varchar(255) NOT NULL COMMENT 'Name of the Virtual Router', + `last_alert_timestamp` varchar(255) NOT NULL COMMENT 'Timestamp of the last alert received from Virtual Router', + PRIMARY KEY (`vm_id`), + CONSTRAINT `fk_virtual_router__id` FOREIGN KEY `fk_virtual_router__id` (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET=utf8 + ALTER TABLE `cloud`.`event` ADD COLUMN `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user'; DROP VIEW IF EXISTS `cloud`.`event_view`; diff --git a/systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh b/systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh new file mode 100644 index 00000000000..e5e8abeffda --- /dev/null +++ b/systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# getRouterAlerts.sh --- Send the alerts from routerServiceMonitor.log to Management Server + +source /root/func.sh + +lock="biglock" +locked=$(getLockFile $lock) +if [ "$locked" != "1" ] +then + exit 1 +fi + +#set -x + +filename=/var/log/routerServiceMonitor.log #Monitor service log file +if [ -n "$1" -a -n "$2" ] +then + reqdateval=$(date -d $1 +"%Y%m%d"); + reqtimeval=$(date -d $2 +"%H%M%S"); +else + reqdateval=0 + reqtimeval=0 +fi +if [ -f $filename ] +then + while read line + do + if [ -n "$line" ]; then + dateval=`echo $line |awk '{print $1}'` + timeval=`echo $line |awk '{print $2}'` + + todate=$(date -d "$dateval" +"%Y%m%d") > /dev/null + totime=$(date -d "$timeval" +"%H%M%S") > /dev/null + if [ "$todate" -gt "$reqdateval" ] > /dev/null + then + if [ -n "$alerts" ]; then alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile + elif [ "$todate" -eq "$reqdateval" ] > /dev/null + then + if [ "$totime" -gt "$reqtimeval" ] > /dev/null + then + if [ -n "$alerts" ]; then alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile + fi + fi + fi + done < $filename +fi +if [ -n "$alerts" ]; then + echo $alerts +else + echo "No Alerts" +fi + +unlock_exit 0 $lock $locked \ No newline at end of file diff --git a/systemvm/patches/debian/config/root/monitorServices.py b/systemvm/patches/debian/config/root/monitorServices.py index 0319ece6b87..c1dfba21a46 100755 --- a/systemvm/patches/debian/config/root/monitorServices.py +++ b/systemvm/patches/debian/config/root/monitorServices.py @@ -25,6 +25,7 @@ from subprocess import * from os import path import time import os +import logging class StatusCodes: SUCCESS = 0 @@ -92,6 +93,8 @@ def raisealert(severity, msg, process_name=None): else: log = '['+severity+']' + " " + msg +"\n" + logging.basicConfig(level=logging.INFO,filename='/var/log/routerServiceMonitor.log',format='%(asctime)s %(message)s') + logging.info(log) msg = 'logger -t monit '+ log pout = Popen(msg, shell=True, stdout=PIPE) diff --git a/test/integration/smoke/test_VirtualRouter_alerts.py b/test/integration/smoke/test_VirtualRouter_alerts.py new file mode 100644 index 00000000000..2333d846f14 --- /dev/null +++ b/test/integration/smoke/test_VirtualRouter_alerts.py @@ -0,0 +1,244 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" P1 tests for alert receiving from VR on service failure in VR +""" +#Import Local Modules +import marvin +from marvin.cloudstackTestCase import * +from marvin.cloudstackAPI import * +from marvin.integration.lib.utils import * +from marvin.integration.lib.base import * +from marvin.integration.lib.common import * +from nose.plugins.attrib import attr +import time + + +_multiprocess_shared_ = True + +class Services: + """Test VM Life Cycle Services + """ + + def __init__(self): + self.services = { + + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended in create account to + # ensure unique username generated each time + "password": "password", + }, + "small": + # Create a small virtual machine instance with disk offering + { + "displayname": "testserver", + "username": "root", # VM creds for SSH + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "service_offerings": + { + "small": + { + # Small service offering ID to for change VM + # service offering from medium to small + "name": "SmallInstance", + "displaytext": "SmallInstance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 256, + }, + "big": + { + # Big service offering ID to for change VM + "name": "BigInstance", + "displaytext": "BigInstance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 512, + } + }, + #Change this + "template": { + "displaytext": "xs", + "name": "xs", + "passwordenabled": False, + }, + "sleep": 60, + "timeout": 10, + #Migrate VM to hostid + "ostype": 'CentOS 5.3 (64-bit)', + # CentOS 5.3 (64-bit) + } + + +class TestVRServiceFailureAlerting(cloudstackTestCase): + @classmethod + def setUpClass(cls): + cls.api_client = super(TestVRServiceFailureAlerting, cls).getClsTestClient().getApiClient() + cls.services = Services().services + + # Get Zone, Domain and templates + domain = get_domain(cls.api_client, cls.services) + cls.zone = get_zone(cls.api_client, cls.services) + cls.services['mode'] = cls.zone.networktype + + template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + # Set Zones and disk offerings ?? + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + # Create account, service offerings, vm. + cls.account = Account.create( + cls.api_client, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offerings"]["small"] + ) + + #create a virtual machine + cls.virtual_machine = VirtualMachine.create( + cls.api_client, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup = [ + cls.small_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + cls.api_client = super(TestVRServiceFailureAlerting, cls).getClsTestClient().getApiClient() + cleanup_resources(cls.api_client, cls._cleanup) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + #Clean up, terminate the created ISOs + cleanup_resources(self.apiclient, self.cleanup) + return + + @attr(hypervisor="xenserver") + @attr(tags=["advanced", "basic"]) + def test_01_VRServiceFailureAlerting(self): + + + if self.zone.networktype == "Basic": + list_router_response = list_routers( + self.apiclient, + listall="true" + ) + else: + list_router_response = list_routers( + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) + self.assertEqual( + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) + router = list_router_response[0] + + hosts = list_hosts( + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) + self.assertEqual( + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) + host = hosts[0] + + self.debug("Router ID: %s, state: %s" % (router.id, router.state)) + + self.assertEqual( + router.state, + 'Running', + "Check list router response for router state" + ) + + alertSubject = "Monitoring Service on VR " + router.name + + if self.apiclient.hypervisor.lower() == 'vmware': + result = get_process_status( + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "service dnsmasq status", + hypervisor=self.apiclient.hypervisor + ) + else: + try: + host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + result = get_process_status( + host.ipaddress, + 22, + host.user, + host.passwd, + router.linklocalip, + "service apache2 stop" + ) + except KeyError: + self.skipTest("Marvin configuration has no host credentials to check router services") + + res = str(result) + self.debug("apache process status: %s" % res) + + time.sleep(300) #wait for 5 minutes meanwhile monitor service on VR starts the apache service + + qresultset = self.dbclient.execute( + "select id from alert where subject = '%s' ORDER BY id DESC LIMIT 1;" \ + % str(alertSubject) + ) + self.assertNotEqual( + len(qresultset), + 0, + "Check DB Query result set" + ) + + return