v3.4

Keep Haproxy service start

app/create_db.py

@@ -308,7 +308,7 @@ def update_db_v_3_31(**kwargs):
 	except sqltool.Error as e:
 		if kwargs.get('silent') != 1:
 			if e.args[0] == 'duplicate column name: ldap_user' or e == " 1060 (42S21): Duplicate column name 'ldap_user' ":
-				print('Updating... go to version 3.3')
+				print('Updating... go to version 3.4')
 			else:
 				print("An error occurred:", e)
 		return False
@@ -317,6 +317,28 @@ def update_db_v_3_31(**kwargs):
 		return True
 	cur.close() 
 	con.close()
+	
+
+def update_db_v_3_4(**kwargs):
+	con, cur = get_cur()
+	sql = """
+	ALTER TABLE `servers` ADD COLUMN active INTEGER NOT NULL DEFAULT 0;
+	"""
+	try:    
+		cur.execute(sql)
+		con.commit()
+	except sqltool.Error as e:
+		if kwargs.get('silent') != 1:
+			if e.args[0] == 'duplicate column name: active' or e == " 1060 (42S21): Duplicate column name 'active' ":
+				print('Updating... go to version 3.4')
+			else:
+				print("An error occurred:", e)
+		return False
+	else:
+		print("DB was update to 3.4<br />")
+		return True
+	cur.close() 
+	con.close()
 			
 def update_all():	
 	update_db_v_31()
@@ -326,6 +348,7 @@ def update_all():
 	update_db_v_3_2_8()
 	update_db_v_3_3()
 	update_db_v_3_31()
+	update_db_v_3_4()
 	
 def update_all_silent():
 	update_db_v_31(silent=1)
@@ -335,6 +358,7 @@ def update_all_silent():
 	update_db_v_3_2_8(silent=1)
 	update_db_v_3_3(silent=1)
 	update_db_v_3_31(silent=1)
+	update_db_v_3_4(silent=1)
 	
 if __name__ == "__main__":
 	create_table()

app/funct.py

@@ -62,6 +62,9 @@ def logging(serv, action, **kwargs):
 	elif kwargs.get('metrics') == 1:
 		mess = get_data('date_in_log') + action + "\n"
 		log = open(log_path + "/metrics-"+get_data('logs')+".log", "a")
+	elif kwargs.get('keep_alive') == 1:
+		mess = get_data('date_in_log') + action + "\n"
+		log = open(log_path + "/keep_alive-"+get_data('logs')+".log", "a")
 	else:
 		mess = get_data('date_in_log') + " from " + IP + " user: " + login + " " + action + " for: " + serv + "\n"
 		log = open(log_path + "/config_edit-"+get_data('logs')+".log", "a")

app/sql.py

@@ -120,11 +120,11 @@ def update_group(name, descript, id):
 	cur.close()    
 	con.close()
 
-def add_server(hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc):
+def add_server(hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc, active):
 	con, cur = create_db.get_cur()
-	sql = """ INSERT INTO servers (hostname, ip, groups, type_ip, enable, master, cred, alert, metrics, port, `desc`) 
-			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')
-		""" % (hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc)
+	sql = """ INSERT INTO servers (hostname, ip, groups, type_ip, enable, master, cred, alert, metrics, port, `desc`, active) 
+			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')
+		""" % (hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc, active)
 	try:    
 		cur.execute(sql)
 		con.commit()
@@ -150,7 +150,7 @@ def delete_server(id):
 	cur.close()    
 	con.close() 		
 
-def update_server(hostname, ip, group, typeip, enable, master, id, cred, alert, metrics, port, desc):
+def update_server(hostname, ip, group, typeip, enable, master, id, cred, alert, metrics, port, desc, active):
 	con, cur = create_db.get_cur()
 	sql = """ update servers set 
 			hostname = '%s',
@@ -163,8 +163,9 @@ def update_server(hostname, ip, group, typeip, enable, master, id, cred, alert,
 			alert = '%s',
 			metrics = '%s',
 			port = '%s',
-			`desc` = '%s'
-			where id = '%s'""" % (hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc, id)
+			`desc` = '%s',
+			active = '%s'
+			where id = '%s'""" % (hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc, active, id)
 	try:    
 		cur.execute(sql)
 		con.commit()
@@ -1137,6 +1138,18 @@ def select_alert(**kwargs):
 		return cur.fetchall()
 	cur.close()    
 	con.close() 
+	
+def select_keep_alive(**kwargs):
+	con, cur = create_db.get_cur()
+	sql = """select ip from servers where active = 1 """
+	try:    
+		cur.execute(sql)
+	except sqltool.Error as e:
+		out_error(e)
+	else:
+		return cur.fetchall()
+	cur.close()    
+	con.close() 
 				
 form = cgi.FieldStorage()
 error_mess = '<span class="alert alert-danger" id="error">All fields must be completed <a title="Close" id="errorMess"><b>X</b></a></span>'
@@ -1187,11 +1200,12 @@ if form.getvalue('newserver') is not None:
 	page = page.split("#")[0]
 	port = form.getvalue('newport')	
 	desc = form.getvalue('desc')	
+	active = form.getvalue('active')	
 	print('Content-type: text/html\n')
 	if ip is None or group is None or cred is None or port is None:		
 		print(error_mess)
 	else:	
-		if add_server(hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc):
+		if add_server(hostname, ip, group, typeip, enable, master, cred, alert, metrics, port, desc, active):
 			show_update_server(ip, page)
 		
 if form.getvalue('serverdel') is not None:
@@ -1238,11 +1252,12 @@ if form.getvalue('updateserver') is not None:
 	metrics = form.getvalue('metrics')	
 	port = form.getvalue('port')	
 	desc = form.getvalue('desc')	
+	active = form.getvalue('active')	
 	print('Content-type: text/html\n')
 	if name is None or ip is None or port is None:
 		print(error_mess)
 	else:		
-		update_server(name, ip, group, typeip, enable, master, id, cred, alert, metrics, port, desc)
+		update_server(name, ip, group, typeip, enable, master, id, cred, alert, metrics, port, desc, active)
 			
 if form.getvalue('updatessh'):
 	id = form.getvalue('id')

app/templates/admin.html

@@ -32,7 +32,13 @@
 			<tr>
 		{% for user in users %}
 			<tr id="user-{{user.0}}" class="{{ loop.cycle('odd', 'even') }}">
-				<td class="padding10 first-collumn"><input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control"></td> 
+				<td class="padding10 first-collumn">
+					{% if user.6 == 1%}
+						<input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control" readonly>
+					{% else %}
+						<input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control">
+					{% endif %}
+				</td> 
 				<td>
 					{% if user.6 == 1%}
 						<input type="password" id="password-{{user.0}}" value="{{user.3}}" class="form-control" readonly>
@@ -40,7 +46,13 @@
 						<input type="password" id="password-{{user.0}}" value="{{user.3}}" class="form-control">
 					{% endif %}
 				</td>
-				<td><input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control"></td>
+				<td>
+					{% if user.6 == 1%}
+						<input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control" readonly>
+					{% else %}
+						<input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control">
+					{% endif %}
+				</td>
 				<td>
 					<select id="role-{{user.0}}" name="role-{{user.0}}">
 						<option disabled selected>Choose role</option>
@@ -135,6 +147,7 @@
 				<td class="checkbox-head"><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
 				<td class="checkbox-head"><span title="Alert if backend change status. Before enable add Telgram chanel at Checker tab">Alert(?)</span></td>
 				<td class="checkbox-head"><span title="Enable save and show metrics">Metrics(?)</span></td>
+				<td class="checkbox-head"><span title="Keep active HAProxy service if down">Active(?)</span></td>
 				<td class="slave-field"><span title="Actions with master config will automatically apply on slave">Slave for (?)</span></td>
 				<td class="cred-field">Credentials</td>
 				<td>Desciption</td>
@@ -192,6 +205,13 @@
 						<label for="metrics-{{server.0}}"></label><input type="checkbox" id="metrics-{{server.0}}">
 					{% endif %}
 				</td>
+				<td class="checkbox">
+					{% if server.11 == 1 %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}" checked>
+					{% else %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option value="0" selected>Not slave</option>
@@ -575,6 +595,12 @@
 					<label for="metrics"></label><input type="checkbox" id="metrics">
 				</td>
 			</tr>
+			<tr>
+			<td class="padding20" title="Keep active HAProxy service if down">Active</td>
+			<td>
+				<label for="active"></label><input type="checkbox" id="active">
+			</td>
+		</tr>
 			<tr>
 				<td class="padding20" title="Actions with master config will automatically apply on slave">Slave for</td>
 				<td>

app/templates/ajax/new_server.html

@@ -53,6 +53,13 @@
 						<label for="metrics-{{server.0}}"></label><input type="checkbox" id="metrics-{{server.0}}">
 					{% endif %}
 				</td>
+				<td>
+					{% if server.11 == 1 %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}" checked>
+					{% else %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option value="0" selected>Not slave</option>

app/templates/base.html

@@ -99,7 +99,7 @@
 					</ul>
 				</nav>
 				<div class="copyright-menu">
-					HAproxy-WI v3.3
+					HAproxy-WI v3.4
 					<br>
 					<a href="https://www.patreon.com/haproxy_wi" title="Donate" target="_blank" style="color: #fff; margin-left: 30px; color: red;" class="patreon">  Patreon</a>
 				</div>

app/templates/servers.html

@@ -33,7 +33,11 @@
 			{% if user.5 == group %}
 			<tr id="user-{{user.0}}" class="{{ loop.cycle('odd', 'even') }}">
 				<td class="padding10 first-collumn">
-					<input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control">
+					{% if user.6 == 1%}
+						<input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control" readonly>
+					{% else %}
+						<input type="text" id="login-{{user.0}}" value="{{user.1}}" class="form-control">
+					{% endif %}
 					<input type="hidden" id="usergroup-{{user.0}}" name="usergroup-{{user.0}}" value="{{ group }}">					
 				</td> 
 				<td>
@@ -43,7 +47,13 @@
 						<input type="password" id="password-{{user.0}}" value="{{user.3}}" class="form-control">
 					{% endif %}
 				</td> 
-				<td><input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control"></td>
+				<td>
+					{% if user.6 == 1%}
+						<input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control" readonly>
+					{% else %}
+						<input type="text" id="email-{{user.0}}" value="{{user.2}}" class="form-control">
+					{% endif %}
+				</td>
 				<td>
 					<select id="role-{{user.0}}" name="role-{{user.0}}">
 						<option disabled selected>Choose role</option>
@@ -79,6 +89,7 @@
 				<td class="checkbox-head"><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
 				<td class="checkbox-head"><span title="Alert if backend change status. Before enable add Telgram chanel at Checker tab">Alert(?)</span></td>
 				<td class="checkbox-head"><span title="Enable save and show metrics">Metrics(?)</span></td>
+				<td class="checkbox-head"><span title="Keep active HAProxy service if down">Active(?)</span></td>
 				<td class="slave-field"><span title="Actions with master config will automatically apply on slave">Slave for (?)</span></td>
 				<td class="cred-field">Credentials</td>
 				<td>Desciption</td>
@@ -126,6 +137,13 @@
 						<label for="metrics-{{server.0}}"></label><input type="checkbox" id="metrics-{{server.0}}">
 					{% endif %}
 				</td>
+				<td class="checkbox">
+					{% if server.11 == 1 %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}" checked>
+					{% else %}
+						<label for="active-{{server.0}}"></label><input type="checkbox" id="active-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option selected value="0">Not slave</option>
@@ -446,6 +464,12 @@
 				<label for="metrics"></label><input type="checkbox" id="metrics">
 			</td>
 		</tr>
+		<tr>
+			<td class="padding20" title="Keep active HAProxy service if down">Active</td>
+			<td>
+				<label for="active"></label><input type="checkbox" id="active">
+			</td>
+		</tr>
 		<tr>
 			<td class="padding20" title="Actions with master config will automatically apply on slave">Slave for</td>
 			<td>

app/tools/checker_master.py

@@ -65,7 +65,7 @@ def get_worker():
 	return output
 	
 if __name__ == "__main__":
-	funct.logging("localhost", " Master started", alerting=1)
+	funct.logging("localhost", " Checker master started", alerting=1)
 	killer = GracefulKiller()
 	
 	while True:
@@ -76,4 +76,4 @@ if __name__ == "__main__":
 			break
 			
 	kill_all_workers()
-	funct.logging("localhost", " Master shutdown", alerting=1)
+	funct.logging("localhost", " Checker master shutdown", alerting=1)

app/tools/checker_worker.py

@@ -52,7 +52,7 @@ def main(serv, port):
 					funct.telegram_send_mess(str(alert), ip=serv)
 					funct.logging("localhost", " "+alert, alerting=1)
 					firstrun = True
-					time.sleep(2)	
+					time.sleep(5)	
 			old_stat_service = cur_stat_service
 			
 		vips = readstats.splitlines()
@@ -102,4 +102,4 @@ if __name__ == "__main__":
 		try:
 			main(args.IP, args.port)
 		except KeyboardInterrupt:
-			pass
+			pass

app/tools/keep_alive.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import check_output, CalledProcessError
+import time
+import os, sys
+sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd())))
+sys.path.append(os.path.join(sys.path[0], os.getcwd()))
+import funct
+import sql
+import signal
+
+class GracefulKiller:
+	kill_now = False
+	def __init__(self):
+		signal.signal(signal.SIGINT, self.exit_gracefully)
+		signal.signal(signal.SIGTERM, self.exit_gracefully)
+	
+	def exit_gracefully(self,signum, frame):
+		self.kill_now = True
+		
+
+def main():
+	servers = sql.select_keep_alive()
+	port = sql.get_setting('haproxy_sock_port')
+	firstrun = True
+	currentstat = []
+	oldstat = []
+	readstats = ""
+	killer = GracefulKiller()
+	old_stat_service = ""
+	cur_stat_service = ""
+	
+	while True:
+		for serv in servers:
+			try:			
+				readstats = subprocess.check_output(["echo show stat | nc "+serv[0]+" "+port], shell=True)
+			except CalledProcessError as e:
+				if firstrun == False:	
+					cur_stat_service = "error"
+					if old_stat_service != cur_stat_service:
+						alert = "Try start HAProxy serivce at " + serv[0]
+						#funct.telegram_send_mess(str(alert), ip=serv[2])
+						funct.logging("localhost", " "+alert, keep_alive=1)
+						start_command = []
+						start_command.append(sql.get_setting('restart_command'))
+						funct.ssh_command(serv[0], start_command)
+
+				firstrun = False				
+				old_stat_service = cur_stat_service
+				continue
+			except OSError as e:
+				print(e)
+				sys.exit()
+			else:
+				cur_stat_service = "Ok"
+				"""
+						
+					if old_stat_service != cur_stat_service:
+						alert = "Now UP HAProxy service at " + serv
+						funct.telegram_send_mess(str(alert), ip=serv[2])
+						funct.logging("localhost", " "+alert, alerting=1)
+							
+						time.sleep(5)	
+				"""
+				if firstrun == False:
+					firstrun = True
+				old_stat_service = cur_stat_service	
+		
+		
+if __name__ == "__main__":
+	funct.logging("localhost", " Keep alive service started", keep_alive=1)
+	killer = GracefulKiller()
+	
+	while True:
+		main()
+		time.sleep(60)
+		
+		if killer.kill_now:
+			break
+			
+	funct.logging("localhost", " Keep alive service shutdown", keep_alive=1)

install.sh

@@ -233,6 +233,46 @@ cat << EOF > /etc/logrotate.d/metrics
 }
 EOF
 
+cat << EOF > /etc/systemd/system/keep_alive.service
+[Unit]
+Description=Keep Alive Haproxy 
+After=syslog.target network.target
+
+[Service]
+Type=simple
+WorkingDirectory=/var/www/$HOME_HAPROXY_WI/app/
+ExecStart=/var/www/$HOME_HAPROXY_WI/app/tools/keep_alive.py
+
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=keep_alive
+
+RestartSec=2s
+Restart=on-failure
+TimeoutStopSec=1s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+cat << 'EOF' > /etc/rsyslog.d/keep_alive.conf 
+if $programname startswith 'keep_alive' then /var/www/__HOME_HAPROXY_WI__/log/keep_alive.log
+& stop
+EOF
+sed -i -e "s/__HOME_HAPROXY_WI__/$HOME_HAPROXY_WI/g" /etc/rsyslog.d/keep_alive.conf
+
+cat << EOF > /etc/logrotate.d/metrics
+/var/www/$HOME_HAPROXY_WI/log/keep_alive.log {
+    daily
+    rotate 10
+    missingok
+    notifempty
+	create 0644 apache apache
+	dateext
+    sharedscripts
+}
+EOF
+
 sed -i 's/#$UDPServerRun 514/$UDPServerRun 514/g' /etc/rsyslog.conf
 sed -i 's/#$ModLoad imudp/$ModLoad imudp/g' /etc/rsyslog.conf
 
@@ -241,8 +281,10 @@ systemctl restart logrotate
 systemctl restart rsyslog
 systemctl restart metrics_haproxy.service
 systemctl restart checker_haproxy.service
+systemctl restart keep_alive.service
 systemctl enable metrics_haproxy.service
 systemctl enable checker_haproxy.service
+systemctl enable keep_alive.service
 
 if hash apt-get 2>/dev/null; then
 	sed -i 's|/var/log/httpd/|/var/log/apache2/|g' $HAPROXY_WI_VHOST_CONF

update.sh

@@ -21,6 +21,49 @@ else
 	yum -y install git nmap-ncat net-tools python35u dos2unix python35u-pip httpd python35u-devel gcc-c++ openldap-devel 
 fi
 
+cat << EOF > /etc/systemd/system/keep_alive.service
+[Unit]
+Description=Keep Alive Haproxy 
+After=syslog.target network.target
+
+[Service]
+Type=simple
+WorkingDirectory=/var/www/$HOME_HAPROXY_WI/app/
+ExecStart=/var/www/$HOME_HAPROXY_WI/app/tools/keep_alive.py
+
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=keep_alive
+
+RestartSec=2s
+Restart=on-failure
+TimeoutStopSec=1s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+cat << 'EOF' > /etc/rsyslog.d/keep_alive.conf 
+if $programname startswith 'keep_alive' then /var/www/__HOME_HAPROXY_WI__/log/keep_alive.log
+& stop
+EOF
+sed -i -e "s/__HOME_HAPROXY_WI__/$HOME_HAPROXY_WI/g" /etc/rsyslog.d/keep_alive.conf
+
+cat << EOF > /etc/logrotate.d/metrics
+/var/www/$HOME_HAPROXY_WI/log/keep_alive.log {
+    daily
+    rotate 10
+    missingok
+    notifempty
+	create 0644 apache apache
+	dateext
+    sharedscripts
+}
+EOF
+
+systemctl restart keep_alive.service
+systemctl enable keep_alive.service
+
 cd app/
 ./create_db.py