pistar-watchdog

#!/usr/bin/python

###############################################################################
#                                                                             #
#                         Pi-Star Service Watchdog                            #
#                                                                             #
#    Version 2.0, Code, Design and Development by Andy Taylor (MW0MWZ).       #
#                                                                             #
#   The service watchdog will monitor the D-Star Services, restarting them    #
#   after any kind of failures - something that used to happen occasionally   #
#             when using the origional G4KLX D-Star software.                 #
#                                                                             #
###############################################################################

import subprocess
import time
import os
import linecache
import datetime
import ConfigParser

time.sleep(90) #Assumiing the Pi has just started up, let the services settle...

# Get the disk usage
def disk_stat(path):
    disk = os.statvfs(path)
    percent = (disk.f_blocks - disk.f_bfree) * 100 / (disk.f_blocks -disk.f_bfree + disk.f_bavail) + 1
    return percent

while True: #Main loop
	# Get the current status
	dstarrepeater = "/etc/dstar-radio.dstarrepeater"
	mmdvmhost = "/etc/dstar-radio.mmdvmhost"

	# Check the disk usage
	if disk_stat('/var/log') > 90:
		# Clean up the log volume
		os.system('rm -rf /var/log/*.1.gz')
		os.system('rm -rf /var/log/*.1')
		os.system('echo "$(tail -500 /var/log/nginx/error.log)" > /var/log/nginx/error.log')
		os.system('rm -f $(find /var/log/pi-star/*.log -type f -mtime +0 -print)')
		os.system('rm -f $(find /var/log/ -type f -mtime +0 -print | grep .gz)')
		# Shorten the Pi-Star logs
		utcnow = datetime.datetime.utcnow()
		datenow = utcnow.strftime('%Y-%m-%d')
		if os.path.isfile(dstarrepeater):
			dstarLogOld = '/var/log/pi-star/DStarRepeater-' + datenow + '.log'
			dstarLogNew = '/var/log/pi-star/dstarrepeaterd-' + datenow + '.log'
			if os.path.isfile(dstarLogOld):
				os.system('echo "$(tail -500 /var/log/pi-star/DStarRepeater-$(date +%F).log)" > /var/log/pi-star/DStarRepeater-$(date +%F).log')
			if os.path.isfile(dstarLogNew):
				os.system('echo "$(tail -500 /var/log/pi-star/dstarrepeaterd-$(date +%F).log)" > /var/log/pi-star/dstarrepeaterd-$(date +%F).log')
			os.system('/bin/systemctl restart dstarrepeater.service')
			time.sleep(60) # Give the service 60 secs before we re-test
		if os.path.isfile(mmdvmhost):
			mmdvmLog = '/var/log/pi-star/MMDVM-' + datenow + '.log'
			if os.path.isfile(mmdvmLog):
				os.system('echo "$(tail -500 /var/log/pi-star/MMDVM-$(date +%F).log)" > /var/log/pi-star/MMDVM-$(date +%F).log')
			os.system('/bin/systemctl restart mmdvmhost.service')
			time.sleep(60) # Give the service 60 secs before we re-test

	# Check the D-Star Specific Services
	if os.path.isfile(dstarrepeater):
		checkprocdstar = subprocess.Popen('pgrep' + ' dstarrepeaterd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdstar.stdout.readlines():
			os.system('/bin/systemctl stop dstarrepeater.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dstarrepeater.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdstar.wait()

		# At this point it's a safe bet that the service is running, now look for signs of DVAP failure.
		utcnow = datetime.datetime.utcnow()
		datenow = utcnow.strftime('%Y-%m-%d')
		dateminus300sec = datetime.datetime.utcnow() - datetime.timedelta(minutes=5)
		logstampnow = utcnow.strftime('%Y-%m-%d %H:%M:%S')
		logstampnowminus300sec = dateminus300sec.strftime('%Y-%m-%d %H:%M:%S')
		# Best figure out the log name now
		currentLogOld = '/var/log/pi-star/DStarRepeater-' + datenow + '.log'
		currentLogNew = '/var/log/pi-star/dstarrepeaterd-' + datenow + '.log'
		if os.path.isfile(currentLogOld):
			currentLog = currentLogOld
		if os.path.isfile(currentLogNew):
			currentLog = currentLogNew

		# Open the DStarRepeater Log
		logfile = open(currentLog, 'r')
		loglist = logfile.readlines()
		logfile.close()

		# Parse the log lines
		for line in loglist:
			# We only care about logs in the last 60 secs
			if line[3:22] >= logstampnowminus300sec and line[3:22] <= logstampnow:
				# Now we look for DVAP failure lines
				if (str('Bad DVAP header') in line) or (str('Cannot open the D-Star modem') in line):
					os.system('/bin/systemctl stop dstarrepeater.service')
					time.sleep(30) # Give the DVAP time to recover
					os.system('/bin/systemctl start dstarrepeater.service')
					time.sleep(270) # Give the service time to start before we move on
				# Look for "No space to write data|the header|end data" and restart the service
				if (str('No space to write') in line):
					os.system('echo "$(tail -n 50 ' + currentLog + ')" > ' + currentLog)
					os.system('/bin/systemctl restart dstarrepeater.service')
					time.sleep(60) # Give the service 60 secs before we re-test

	# Check the MMDVMHost Specific Services
	if os.path.isfile(mmdvmhost):
		checkprocmmdvm = subprocess.Popen('pgrep' + ' MMDVMHost', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocmmdvm.stdout.readlines():
			os.system('/bin/systemctl stop mmdvmhost.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start mmdvmhost.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocmmdvm.wait()
		
		checkprocdmrgateway = subprocess.Popen('pgrep' + ' DMRGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdmrgateway.stdout.readlines():
			os.system('/bin/systemctl stop dmrgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmrgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmrgateway.wait()

		checkprocysfgateway = subprocess.Popen('pgrep' + ' YSFGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocysfgateway.stdout.readlines():
			os.system('/bin/systemctl stop ysfgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysfgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysfgateway.wait()

		checkprocdgidgateway = subprocess.Popen('pgrep' + ' DGIdGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdgidgateway.stdout.readlines():
			os.system('/bin/systemctl stop dgidgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dgidgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdgidgateway.wait()

		checkprocp25gateway = subprocess.Popen('pgrep' + ' P25Gateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocp25gateway.stdout.readlines():
			os.system('/bin/systemctl stop p25gateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start p25gateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocp25gateway.wait()

		checkprocm17gateway = subprocess.Popen('pgrep' + ' M17Gateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocm17gateway.stdout.readlines():
			os.system('/bin/systemctl stop m17gateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start m17gateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocm17gateway.wait()

		checkprocnxdngateway = subprocess.Popen('pgrep' + ' NXDNGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocnxdngateway.stdout.readlines():
			os.system('/bin/systemctl stop nxdngateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start nxdngateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocnxdngateway.wait()

		checkprocysfparrot = subprocess.Popen('pgrep' + ' YSFParrot', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocysfparrot.stdout.readlines():
			os.system('/bin/systemctl stop ysfparrot.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysfparrot.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysfparrot.wait()

		checkprocp25parrot = subprocess.Popen('pgrep' + ' P25Parrot', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocp25parrot.stdout.readlines():
			os.system('/bin/systemctl stop p25parrot.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start p25parrot.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocp25parrot.wait()

		checkprocnxdnparrot = subprocess.Popen('pgrep' + ' NXDNParrot', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocnxdnparrot.stdout.readlines():
			os.system('/bin/systemctl stop nxdnparrot.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start nxdnparrot.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocnxdnparrot.wait()

		checkprocysf2dmr = subprocess.Popen('pgrep' + ' YSF2DMR', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocysf2dmr.stdout.readlines():
			os.system('/bin/systemctl stop ysf2dmr.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2dmr.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2dmr.wait()

		checkprocysf2nxdn = subprocess.Popen('pgrep' + ' YSF2NXDN', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocysf2nxdn.stdout.readlines():
			os.system('/bin/systemctl stop ysf2nxdn.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2nxdn.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2nxdn.wait()

		checkprocysf2p25 = subprocess.Popen('pgrep' + ' YSF2P25', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocysf2p25.stdout.readlines():
			os.system('/bin/systemctl stop ysf2p25.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start ysf2p25.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocysf2p25.wait()

		checkprocdmr2ysf = subprocess.Popen('pgrep' + ' DMR2YSF', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdmr2ysf.stdout.readlines():
			os.system('/bin/systemctl stop dmr2ysf.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmr2ysf.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmr2ysf.wait()

		checkprocdmr2nxdn = subprocess.Popen('pgrep' + ' DMR2NXDN', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdmr2nxdn.stdout.readlines():
			os.system('/bin/systemctl stop dmr2nxdn.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dmr2nxdn.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdmr2nxdn.wait()

		checkprocdapnetgateway = subprocess.Popen('pgrep' + ' DAPNETGateway', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		if not checkprocdapnetgateway.stdout.readlines():
			os.system('/bin/systemctl stop dapnetgateway.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start dapnetgateway.service')
			time.sleep(10) #Give the service time to start before we move on
		checkprocdapnetgateway.wait()

	# Check the services that can be used by all systems
	checkprocirc = subprocess.Popen('pgrep' + ' ircddbgatewayd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	if not checkprocirc.stdout.readlines():
		os.system('/bin/systemctl stop ircddbgateway.service')
		time.sleep(2) #Give the service time to stop before we move on
		os.system('/bin/systemctl start ircddbgateway.service')
		time.sleep(10) #Give the service time to start before we move on
	checkprocirc.wait()

	checkproctime = subprocess.Popen('pgrep' + ' timeserverd', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	if not checkproctime.stdout.readlines():
		os.system('/bin/systemctl stop timeserver.service')
		time.sleep(2) #Give the service time to stop before we move on
		os.system('/bin/systemctl start timeserver.service')
		time.sleep(10) #Give the service time to start before we move on
	checkproctime.wait()

	# If PiStar-Remote not enabled, dont try and keep it running
	pistarRemoteConfig = ConfigParser.RawConfigParser()
	pistarRemoteConfig.read('/etc/pistar-remote')

	isEnabled = pistarRemoteConfig.get('enable', 'enabled')
	if (isEnabled == 'true'):
		checkprocremote = subprocess.check_output(['/usr/local/sbin/pistar-remote.service', 'status'])
		if "not" in checkprocremote:
			os.system('/bin/systemctl stop pistar-remote.service')
			time.sleep(2) #Give the service time to stop before we move on
			os.system('/bin/systemctl start pistar-remote.service')
			time.sleep(10) #Give the service time to start before we move on

#	checkprockeeper = subprocess.Popen('pgrep' + ' pistar-keeper', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
#	if not checkprockeeper.stdout.readlines():
#		os.system('service pistar-keeper start')
#		time.sleep(10) #Give the service time to start before we move on
#	checkprockeeper.wait()

	time.sleep(300) #Time to wait between service checks