#!/usr/bin/python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from configparser import ConfigParser from subprocess import * from datetime import datetime import time import os import logging import json from os import sys, path from health_checks.utility import getHealthChecksData class StatusCodes: SUCCESS = 0 FAILED = 1 INVALID_INP = 2 RUNNING = 3 STOPPED = 4 STARTING = 5 class Log: INFO = 'INFO' ALERT = 'ALERT' CRIT = 'CRIT' NOTIF = 'NOTIF' class Config: SLEEP_SEC = 1 RETRY_ITERATIONS = 10 RETRY_FOR_RESTART = 5 MONITOR_LOG = '/var/log/monitor.log' HEALTH_CHECKS_DIR = 'health_checks' MONITOR_RESULT_FILE_SUFFIX = 'monitor_results.json' FAILING_CHECKS_FILE = 'failing_health_checks' def getServicesConfig( config_file_path = "/etc/monitor.conf" ): """ Reads the process configuration from the config file. Config file contains the processes to be monitored. """ process_dict = {} parser = ConfigParser() parser.read( config_file_path ) for section in parser.sections(): process_dict[section] = {} for name, value in parser.items(section): process_dict[section][name] = value printd (" %s = %r" % (name, value)) return process_dict def printd (msg): """ prints the debug messages """ #for debug #print msg f= open(Config.MONITOR_LOG, 'w' if not path.isfile(Config.MONITOR_LOG) else 'r+') f.seek(0, 2) f.write(str(msg)+"\n") f.close() print(str(msg)) def raisealert(severity, msg, process_name=None): """ Writes the alert message""" #timeStr=str(time.ctime()) if process_name is not None: log = '['+severity +']'+" " + '['+process_name+']' + " " + msg +"\n" else: log = '['+severity+']' + " " + msg +"\n" logging.basicConfig(level=logging.INFO,filename='/var/log/routerServiceMonitor.log',format='%(asctime)s %(message)s') logging.info(log) msg = 'logger -t monit '+ log pout = Popen(msg, shell=True, stdout=PIPE) print("[Alert] " + msg) def isPidMatchPidFile(pidfile, pids): """ Compares the running process pid with the pid in pid file. If a process with multiple pids then it matches with pid file """ if pids is None or isinstance(pids,list) != True or len(pids) == 0: printd ("Invalid Arguments") return StatusCodes.FAILED if not path.isfile(pidfile): #It seems there is no pid file for this service printd("The pid file "+pidfile+" is not there for this process") return StatusCodes.FAILED fd=None try: fd = open(pidfile,'r') except: printd("pid file: "+ pidfile +" open failed") return StatusCodes.FAILED inp = fd.read() if not inp: fd.close() return StatusCodes.FAILED printd("file content of pidfile " + pidfile + " = " + str(inp).strip()) printd(pids) tocheck_pid = inp.strip() for item in pids: if str(tocheck_pid) == item.strip(): printd("pid file matched") fd.close() return StatusCodes.SUCCESS fd.close() return StatusCodes.FAILED def checkProcessRunningStatus(process_name, pidFile): printd("checking the process " + process_name) cmd = '' pids = [] cmd = 'pidof ' + process_name printd(cmd) #cmd = 'service ' + process_name + ' status' pout = Popen(cmd, shell=True, stdout=PIPE) exitStatus = pout.wait() temp_out = pout.communicate()[0].decode() #check there is only one pid or not if exitStatus == 0: pids = temp_out.strip().split(' ') printd("pid(s) of process %s are %s " %(process_name, pids)) #there is more than one process so match the pid file #if not matched set pidFileMatched=False printd("Checking pid file") if isPidMatchPidFile(pidFile, pids) == StatusCodes.SUCCESS: return True,pids printd("pid of exit status %s" %exitStatus) return False,pids def restartService(service_name): cmd = 'service ' + service_name + ' restart' cout = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT) return_val = cout.wait() if return_val == 0: printd("The service " + service_name +" recovered successfully ") msg="The process " +service_name+" is recovered successfully " raisealert(Log.INFO,msg,service_name) return True else: printd("process restart failed ....") return False def checkProcessStatus( process ): """ Check the process running status, if not running tries to restart Returns the process status and if it was restarted """ process_name = process.get('processname') service_name = process.get('servicename') pidfile = process.get('pidfile') #temp_out = None restartFailed=False pidFileMatched=False pids='' cmd='' if process_name is None: printd ("\n Invalid Process Name") return StatusCodes.INVALID_INP, False status, pids = checkProcessRunningStatus(process_name, pidfile) if status == True: printd("The process is running ....") return StatusCodes.RUNNING, False else: printd("Process %s is not running trying to recover" %process_name) #Retry the process state for few seconds for i in range(1, Config.RETRY_ITERATIONS): time.sleep(Config.SLEEP_SEC) if i < Config.RETRY_FOR_RESTART: # this is just for trying few more times status, pids = checkProcessRunningStatus(process_name, pidfile) if status == True: raisealert(Log.ALERT, "The process detected as running", process_name) break else: printd("Process %s is not running checking the status again..." %process_name) continue else: msg="The process " +process_name+" is not running trying recover " raisealert(Log.INFO,process_name,msg) if service_name == 'apache2': # Killing apache2 process with this the main service will not start for pid in pids: cmd = 'kill -9 '+pid printd(cmd) Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT) if restartService(service_name) == True: break else: restartFailed = True continue #for end here if restartFailed == True: msg="The process %s recover failed "%process_name raisealert(Log.ALERT,process_name,msg) printd("Restart failed after number of retries") return StatusCodes.STOPPED, False return StatusCodes.RUNNING, True def monitProcess( processes_info ): """ Monitors the processes which got from the config file """ checkStartTime = time.time() service_status = {} failing_services = [] if len( processes_info ) == 0: printd("No config items provided - means a redundant VR or a VPC Router") return service_status, failing_services print("[Process Info] " + json.dumps(processes_info)) #time for noting process down time csec = repr(time.time()).split('.')[0] for process,properties in list(processes_info.items()): printd ("---------------------------\nchecking the service %s\n---------------------------- " %process) serviceName = process + ".service" processStatus, wasRestarted = checkProcessStatus(properties) if processStatus != StatusCodes.RUNNING: printd( "\n Service %s is not Running"%process) checkEndTime = time.time() service_status[serviceName] = { "success": "false", "lastUpdate": str(int(checkStartTime * 1000)), "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), "message": "service down at last check " + str(csec) } failing_services.append(serviceName) else: checkEndTime = time.time() service_status[serviceName] = { "success": "true", "lastUpdate": str(int(checkStartTime * 1000)), "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), "message": "service is running" + (", was restarted" if wasRestarted else "") } return service_status, failing_services def execute(script, checkType = "basic"): checkStartTime = time.time() cmd = "./" + script + " " + checkType printd ("Executing health check script command: " + cmd) pout = Popen(cmd, shell=True, stdout=PIPE) exitStatus = pout.wait() output = pout.communicate()[0].decode().strip() checkEndTime = time.time() if exitStatus == 0: if len(output) > 0: printd("Successful execution of " + script) return { "success": "true", "lastUpdate": str(int(checkStartTime * 1000)), "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), "message": output } return {} #Skip script if no output is received else: printd("Script execution failed " + script) return { "success": "false", "lastUpdate": str(int(checkStartTime * 1000)), "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), "message": output } def main(checkType = "basic"): startTime = time.time() ''' Step1 : Get Services Config ''' printd("monitoring started") configDict = getServicesConfig() ''' Step2: Monitor services and Raise Alerts ''' monitResult = {} failingChecks = [] if checkType == "basic": monitResult, failingChecks = monitProcess(configDict) ''' Step3: Run health check scripts as needed ''' hc_data = getHealthChecksData() if hc_data is not None and "health_checks_enabled" in hc_data and hc_data['health_checks_enabled']: hc_exclude = hc_data["excluded_health_checks"] if "excluded_health_checks" in hc_data else [] for f in os.listdir(Config.HEALTH_CHECKS_DIR): if f in hc_exclude: continue fpath = path.join(Config.HEALTH_CHECKS_DIR, f) if path.isfile(fpath) and os.access(fpath, os.X_OK): ret = execute(fpath, checkType) if len(ret) == 0: continue if "success" in ret and ret["success"].lower() == "false": failingChecks.append(f) monitResult[f] = ret ''' Step4: Write results to the json file for admins/management server to read ''' endTime = time.time() monitResult["lastRun"] = { "start": str(datetime.fromtimestamp(startTime)), "end": str(datetime.fromtimestamp(endTime)), "duration": str(endTime - startTime) } with open(checkType + "_" + Config.MONITOR_RESULT_FILE_SUFFIX, 'w') as f: json.dump(monitResult, f, ensure_ascii=False) failChecksFile = checkType + "_" + Config.FAILING_CHECKS_FILE if len(failingChecks) > 0: fcs = "" for fc in failingChecks: fcs = fcs + fc + "," fcs = fcs[0:-1] with open(failChecksFile, 'w') as f: f.write(fcs) elif path.isfile(failChecksFile): os.remove(failChecksFile) if __name__ == "__main__": checkType = "basic" if len(sys.argv) == 2: if sys.argv[1] == "advanced": main("advanced") elif sys.argv[1] == "basic": main("basic") else: printd("Error: Unknown type of test: " + sys.argv) else: main("basic") main("advanced")