apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-5164 Unmonit for 30 minutes for a failed process

Browse Source
This commit is contained in:
Jayapal 2013-11-25 14:58:12 +05:30
parent ab2c38c050
commit 0be4a685e8
2 changed files with 163 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
systemvm/patches/debian/config/opt/cloud/bin/monitor_service.sh
View File

@ -64,7 +64,7 @@ crontab -l | grep -v monitorServices.py | crontab -
create_config $config
#add cron job
(crontab -l ;echo -e "SHELL=/bin/bash\nPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n */1 * * * * /usr/bin/python /root/monitorServices.py") | crontab -
(crontab -l ;echo -e "SHELL=/bin/bash\nPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n */3 * * * * /usr/bin/python /root/monitorServices.py") | crontab -
unlock_exit 0 $lock $locked

203
systemvm/patches/debian/config/root/monitorServices.py
View File

@ -19,14 +19,13 @@
__author__ = 'jayapalreddy'
from ConfigParser import SafeConfigParser
from subprocess import *
from os import path
import time
import os
monitor_log='/var/log/monitor.log'
class StatusCodes:
SUCCESS = 0
FAILED = 1
@ -35,42 +34,58 @@ class StatusCodes:
STOPPED = 4
STARTING = 5
class log:
class Log:
INFO = 'INFO'
ALERT = 'ALERT'
CRIT = 'CRIT'
NOTIF = 'NOTIF'
class Config:
MONIT_AFTER_MINS = 30
SLEEP_SEC = 1
RETRY_ITERATIONS = 10
RETRY_FOR_RESTART = 5
MONITOR_LOG = '/var/log/monitor.log'
UNMONIT_PS_FILE = '/etc/unmonit_psList.txt'
def getConfig( config_file_path = "/etc/monitor.conf" ):
"""
Reads the process configuration from the config file.
Config file contains the processes to be monitored.
"""
process_dict = {}
parser = SafeConfigParser()
parser.read( config_file_path )
#print 'Read values:\n'
for section in parser.sections():
# print section
process_dict[section] = {}
for name, value in parser.items(section):
process_dict[section][name] = value
# print ' %s = %r' % (name, value)
# printd (" %s = %r" % (name, value))
return process_dict
def printd (msg):
"""
prints the debug messages
"""
#for debug
#print msg
return 0
f= open(monitor_log,'r+')
f= open(Config.MONITOR_LOG,'r+')
f.seek(0, 2)
f.write(str(msg)+"\n")
f.close()
def raisealert(severity, msg, process_name=None):
""" Writes the alert message"""
#timeStr=str(time.ctime())
if process_name is not None:
log = '['+severity +']'+" " + '['+process_name+']' + " " + msg +"\n"
@ -82,9 +97,12 @@ def raisealert(severity, msg, process_name=None):
def isPidMatchPidFile(pidfile, pids):
""" Compares the running process pid with the pid in pid file.
If a process with multiple pids then it matches with pid file
"""
if pids is None or isinstance(pids,list) != True or len(pids) == 0:
print "Invalid Arguments"
printd ("Invalid Arguments")
return StatusCodes.FAILED
if not path.isfile(pidfile):
#It seems there is no pid file for this service
@ -100,12 +118,18 @@ def isPidMatchPidFile(pidfile, pids):
inp = fd.read()
if not inp:
fd.close()
return StatusCodes.FAILED
printd("file content "+str(inp))
printd(pids)
tocheck_pid = inp.strip()
for item in pids:
if str(tocheck_pid) == item.strip():
printd("pid file matched")
fd.close()
return StatusCodes.SUCCESS
fd.close()
@ -114,19 +138,22 @@ def isPidMatchPidFile(pidfile, pids):
def checkProcessStatus( process ):
"""
Check the process running status, if not running tries to restart
"""
process_name = process.get('processname')
service_name = process.get('servicename')
pidfile = process.get('pidfile')
#temp_out = None
restartFailed=False
pidFileMatched=1
pidFileMatched=False
pids=''
cmd=''
if process_name is None:
print "\n Invalid Process Name"
printd ("\n Invalid Process Name")
return StatusCodes.INVALID_INP
else:
msg="checking the process " + process_name
printd(msg)
printd("checking the process " + process_name)
cmd = 'pidof ' + process_name
printd(cmd)
#cmd = 'service ' + process_name + ' status'
@ -136,20 +163,19 @@ def checkProcessStatus( process ):
#check there is only one pid or not
if exitStatus == 0:
pids = temp_out.split(' ')
msg="pids: " +temp_out;
printd(msg)
pids = temp_out.split(' ')
#there is more than one process so match the pid file
#if not matched set pidFileMatched=0
#if not matched set pidFileMatched=False
printd("Checking pid file")
if isPidMatchPidFile(pidfile, pids) == StatusCodes.SUCCESS:
pidFileMatched = 1;
pidFileMatched = True;
else:
pidFileMatched = 0;
pidFileMatched = False;
printd(pidFileMatched)
if exitStatus == 0 and pidFileMatched == 1:
if exitStatus == 0 and pidFileMatched == True:
printd("The process is running ....")
return StatusCodes.RUNNING
else:
@ -157,28 +183,28 @@ def checkProcessStatus( process ):
msg="The process " + process_name +" is not running trying recover "
printd(msg)
#Retry the process state for few seconds
for i in range(1,10):
for i in range(1, Config.RETRY_ITERATIONS):
pout = Popen(cmd, shell=True, stdout=PIPE)
exitStatus = pout.wait()
temp_out = pout.communicate()[0]
if i < 5: # this is just for trying few more times
if i < Config.RETRY_FOR_RESTART: # this is just for trying few more times
if exitStatus == 0:
pids = temp_out.split(' ')
if isPidMatchPidFile(pidfile, pids) == StatusCodes.SUCCESS:
pidFileMatched = 1;
pidFileMatched = True;
printd("pid file is matched ...")
raisealert(log.ALERT, "The process detected as running", process_name)
raisealert(Log.ALERT, "The process detected as running", process_name)
break
else:
printd("pid file is not matched ...")
pidFileMatched = 0;
pidFileMatched = False;
time.sleep(Config.SLEEP_SEC)
continue
time.sleep(1)
else:
msg="The process " +process_name+" is not running trying recover "
raisealert(log.INFO,process_name,msg)
raisealert(Log.INFO,process_name,msg)
if service_name == 'apache2':
# Killing apache2 process with this the main service will not start
@ -189,7 +215,7 @@ def checkProcessStatus( process ):
cmd = 'service ' + service_name + ' restart'
time.sleep(1)
time.sleep(Config.SLEEP_SEC)
#return_val= check_call(cmd , shell=True)
cout = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
@ -198,37 +224,135 @@ def checkProcessStatus( process ):
if return_val == 0:
printd("The process" + process_name +" recovered successfully ")
msg="The process " +process_name+" is recovered successfully "
raisealert(log.INFO,msg,process_name)
raisealert(Log.INFO,msg,process_name)
break;
else:
#retry restarting the process for few tries
printd("process restart failing trying again ....")
restartFailed=True
time.sleep(1)
time.sleep(Config.SLEEP_SEC)
continue
#for end here
if restartFailed == True:
msg="The process %s recover failed "%process_name
raisealert(log.ALERT,process_name,msg)
raisealert(Log.ALERT,process_name,msg)
printd("Restart failed after number of retries")
return StatusCodes.STOPPED
return StatusCodes.RUNNING
def raiseAlert( process_name ):
print "process name %s is raised "%process_name
def monitProcess( processes_info ):
"""
Monitors the processes which got from the config file
"""
if len( processes_info ) == 0:
print "Invalid Input"
printd("Invalid Input")
return StatusCodes.INVALID_INP
for process,properties in processes_info.items():
if checkProcessStatus( properties) != StatusCodes.RUNNING:
print "\n Process %s is not Running"%process
dict_unmonit={}
umonit_update={}
if not path.isfile(Config.UNMONIT_PS_FILE):
printd('Unmonit File not exist')
else:
#load the dictionary with unmonit process list
dict_unmonit = loadPsFromUnMonitFile()
#time for noting process down time
csec = repr(time.time()).split('.')[0]
unMonitPs=False
for process,properties in processes_info.items():
#skip the process it its time stamp less than Config.MONIT_AFTER_MINS
printd ("checking the process %s \n" %process)
if not is_emtpy(dict_unmonit):
if dict_unmonit.has_key(process):
ts = dict_unmonit[process]
printd("Time difference=%s" %str(int(csec) - int(ts)))
tmin = (int(csec) - int(ts) )/60
if ( int(csec) - int(ts) )/60 < Config.MONIT_AFTER_MINS:
raisealert(Log.ALERT, "The %s get monitor after %s minutes " %(process, Config.MONIT_AFTER_MINS))
printd('process will be monitored after %s min' %(str(int(Config.MONIT_AFTER_MINS) - tmin)))
unMonitPs=True
continue
if checkProcessStatus( properties) != StatusCodes.RUNNING:
printd( "\n Process %s is not Running"%process)
#add this process into unmonit list
printd ("updating the process for unmonit %s\n" %process)
umonit_update[process]=csec
#if dict is not empty write to file else delete it
if not is_emtpy(umonit_update):
writePsListToUnmonitFile(umonit_update)
else:
if is_emtpy(umonit_update) and unMonitPs == False:
#delete file it is there
if path.isfile(Config.UNMONIT_PS_FILE):
printd("Removing the file %s" %Config.UNMONIT_PS_FILE)
os.remove(Config.UNMONIT_PS_FILE)
def loadPsFromUnMonitFile():
dict_unmonit = {}
try:
fd = open(Config.UNMONIT_PS_FILE)
except:
printd("Failed to open file %s " %(Config.UNMONIT_PS_FILE))
return StatusCodes.FAILED
ps = fd.read()
if not ps:
printd("File %s content is empty " %Config.UNMONIT_PS_FILE)
return StatusCodes.FAILED
printd(ps)
plist = ps.split(',')
plist.remove('')
for i in plist:
dict_unmonit[i.split(':')[0]] = i.split(':')[1]
fd.close();
return dict_unmonit;
def writePsListToUnmonitFile(umonit_update):
printd("Write updated unmonit list to file")
line=''
for i in umonit_update:
line+=str(i)+":"+str(umonit_update[i])+','
printd(line)
try:
fd=open(Config.UNMONIT_PS_FILE,'w')
except:
printd("Failed to open file %s " %Config.UNMONIT_PS_FILE)
return StatusCodes.FAILED
fd.write(line);
fd.close()
def is_emtpy(struct):
"""
Checks wether the given struct is empty or not
"""
if struct:
return False
else:
return True
def main():
'''
@ -238,14 +362,11 @@ def main():
printd("monitoring started")
temp_dict = getConfig()
'''
Step2: Get Previous Run Log
'''
'''
Step3: Monitor and Raise Alert
Step2: Monitor and Raise Alert
'''
#raisealert(log.INFO, 'Monit started')
#raisealert(Log.INFO, 'Monit started')
monitProcess( temp_dict )