v2.7

Alerting about changes backends state

README.md

@@ -27,7 +27,7 @@ Support the project
 17.	SSL certificate support.
 18.	SSH Key support for managing multiple HAproxy Servers straight from haproxy-wi
 19. SYN flood protect
-
+20. Alerting about changes backends state
 
 # Install
 The installer will ask you a few questions

app/create_db.py

@@ -404,7 +404,7 @@ def update_db_v_2_6_1(**kwargs):
 	except sqltool.Error as e:
 		if kwargs.get('silent') != 1:
 			if e.args[0] == 'duplicate column name: groups' or e == "1060 (42S21): Duplicate column name 'groups' ":
-				print('DB was updated. No more run')
+				print('Updating... go to version 2.7')
 			else:
 				print("An error occurred:", e)
 		return False
@@ -414,6 +414,27 @@ def update_db_v_2_6_1(**kwargs):
 	cur.close() 
 	con.close()
 	
+def update_db_v_2_7(**kwargs):
+	con, cur = get_cur()
+	sql = """
+	ALTER TABLE `servers` ADD COLUMN alert INTEGER NOT NULL DEFAULT 0;
+	"""
+	try:    
+		cur.execute(sql)
+		con.commit()
+	except sqltool.Error as e:
+		if kwargs.get('silent') != 1:
+			if e.args[0] == 'duplicate column name: groups' or e == "1060 (42S21): Duplicate column name 'groups' ":
+				print('DB was updated. No more run')
+			else:
+				print("An error occurred:", e)
+		return False
+	else:
+		print("DB was update to 2.7<br />")
+		return True
+	cur.close() 
+	con.close()
+	
 def update_all():	
 	update_db_v_2_0_1()
 	update_db_v_2_0_1_1()
@@ -425,6 +446,7 @@ def update_all():
 	update_db_v_2_6()
 	update_db_v_2_61()
 	update_db_v_2_6_1()
+	update_db_v_2_7()
 	
 def update_all_silent():
 	update_db_v_2_0_1(silent=1)
@@ -437,4 +459,5 @@ def update_all_silent():
 	update_db_v_2_6(silent=1)
 	update_db_v_2_61(silent=1)
 	update_db_v_2_6_1(silent=1)
+	update_db_v_2_7(silent=1)
 		

app/funct.py

@@ -38,17 +38,25 @@ def get_data(type):
 		
 	return now_utc.strftime(fmt)
 			
-def logging(serv, action):
+def logging(serv, action, **kwargs):
 	import sql
-	IP = cgi.escape(os.environ["REMOTE_ADDR"])
-	cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
-	user_uuid = cookie.get('uuid')
-	login = sql.get_user_name_by_uuid(user_uuid.value)
-	mess = get_data('date_in_log') + " from " + IP + " user: " + login + " " + action + " for: " + serv + "\n"
 	log_path = get_config_var('main', 'log_path')
-	
+	try:
-	try:		
+		IP = cgi.escape(os.environ["REMOTE_ADDR"])
+		cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
+		user_uuid = cookie.get('uuid')
+		login = sql.get_user_name_by_uuid(user_uuid.value)
+	except:
+		pass
+		
+	if kwargs.get('alerting') != 1:
+		mess = get_data('date_in_log') + " from " + IP + " user: " + login + " " + action + " for: " + serv + "\n"
 		log = open(log_path + "/config_edit-"+get_data('logs')+".log", "a")
+	else:
+		mess = get_data('date_in_log') + action + "\n"
+		log = open(log_path + "/checker-"+get_data('logs')+".log", "a")
+			
+	try:				
 		log.write(mess)
 		log.close
 	except IOError:
@@ -58,14 +66,15 @@ def logging(serv, action):
 	if get_config_var('telegram', 'enable') == "1": telegram_send_mess(mess)
 
 def telegram_send_mess(mess):
-	import telegram
+	import telebot
+	from telebot import apihelper
 	token_bot = get_config_var('telegram', 'token')
 	channel_name = get_config_var('telegram', 'channel_name')
 	proxy = get_config_var('main', 'proxy')
 	
 	if proxy is not None:
-		pp = telegram.utils.request.Request(proxy_url=proxy)
+		apihelper.proxy = {'https': proxy}
-	bot = telegram.Bot(token=token_bot, request=pp)
+	bot = telebot.TeleBot(token=token_bot)
 	bot.send_message(chat_id=channel_name, text=mess)
 	
 def check_login(**kwargs):

app/sql.py

@@ -110,9 +110,13 @@ def update_group(name, descript, id):
 	cur.close()    
 	con.close()
 
-def add_server(hostname, ip, group, typeip, enable, master, cred):
+def add_server(hostname, ip, group, typeip, enable, master, cred, alert):
 	con, cur = create_db.get_cur()
-	sql = """INSERT INTO servers (hostname, ip, groups, type_ip, enable, master, cred) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s')""" % (hostname, ip, group, typeip, enable, master, cred)
+	sql = """
+			INSERT INTO servers 
+			(hostname, ip, groups, type_ip, enable, master, cred, alert) 
+			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')
+		""" % (hostname, ip, group, typeip, enable, master, cred, alert)
 	try:    
 		cur.execute(sql)
 		con.commit()
@@ -139,7 +143,7 @@ def delete_server(id):
 	cur.close()    
 	con.close() 		
 
-def update_server(hostname, ip, group, typeip, enable, master, id, cred):
+def update_server(hostname, ip, group, typeip, enable, master, id, cred, alert):
 	con, cur = create_db.get_cur()
 	sql = """update servers set 
 			hostname = '%s',
@@ -148,8 +152,9 @@ def update_server(hostname, ip, group, typeip, enable, master, id, cred):
 			type_ip = '%s',
 			enable = '%s',
 			master = '%s',
-			cred = '%s'
+			cred = '%s',
-			where id = '%s'""" % (hostname, ip, group, typeip, enable, master, cred, id)
+			alert = '%s'
+			where id = '%s'""" % (hostname, ip, group, typeip, enable, master, cred, alert, id)
 	try:    
 		cur.execute(sql)
 		con.commit()
@@ -589,6 +594,18 @@ def select_roles(**kwargs):
 		return cur.fetchall()
 	cur.close()    
 	con.close()  
+	
+def select_alert(**kwargs):
+	con, cur = create_db.get_cur()
+	sql = """select ip from servers where alert = 1 """
+	try:    
+		cur.execute(sql)
+	except sqltool.Error as e:
+		print("An error occurred:", e.args[0])
+	else:
+		return cur.fetchall()
+	cur.close()    
+	con.close() 
 			
 form = cgi.FieldStorage()
 error_mess = '<span class="alert alert-danger" id="error">All fields must be completed <a title="Close" id="errorMess"><b>X</b></a></span>'
@@ -635,12 +652,13 @@ if form.getvalue('newserver') is not None:
 	enable = form.getvalue('enable')
 	master = form.getvalue('slave')
 	cred = form.getvalue('cred')
+	alert = form.getvalue('alert_en')
 	if ip is None or group is None or cred is None:
 		print('Content-type: text/html\n')
 		print(error_mess)
 	else:	
 		print('Content-type: text/html\n')
-		if add_server(hostname, ip, group, typeip, enable, master, cred):
+		if add_server(hostname, ip, group, typeip, enable, master, cred, alert):
 			show_update_server(ip)
 		
 if form.getvalue('serverdel') is not None:
@@ -681,13 +699,14 @@ if form.getvalue('updateserver') is not None:
 	master = form.getvalue('slave')		
 	id = form.getvalue('id')	
 	cred = form.getvalue('cred')	
+	alert = form.getvalue('alert_en')	
 	if name is None or ip is None:
 		print('Content-type: text/html\n')
 		print(error_mess)
 	else:		
 		print('Content-type: text/html\n')
 		#if funct.ssh_connect(ip, check=1):
-		update_server(name, ip, group, typeip, enable, master, id, cred)
+		update_server(name, ip, group, typeip, enable, master, id, cred, alert)
 		#else:
 		#	print('<span class="alert alert-danger" id="error"><a title="Close" id="errorMess"><b>X</b></a></span>')
 			

app/templates/admin.html

@@ -163,6 +163,7 @@
 				<td>Group</td>
 				<td>Enable</td>
 				<td><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
+				<td><span title="Alert if backend change status">Alert(?)</span></td>
 				<td><span title="Actions with master config will automatically apply on slave">Slave for (?)</span></td>
 				<td>Credentials</td>
 				<td></td>
@@ -202,6 +203,13 @@
 						<label for="typeip-{{server.0}}"></label><input type="checkbox" id="typeip-{{server.0}}">
 					{% endif %}
 				</td>
+				<td>
+					{% if server.8 == 1 %}
+						<label for="alert-{{server.0}}"></label><input type="checkbox" id="alert-{{server.0}}" checked>
+					{% else %}
+						<label for="alert-{{server.0}}"></label><input type="checkbox" id="alert-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option value="0" selected>Not slave</option>
@@ -240,7 +248,8 @@
 				<td>IP</td>
 				<td>Group</td>
 				<td>Enable</td>
-				<td>Virt</td>
+				<td><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
+				<td><span title="Alert if backend change status">Alert(?)</span></td>
 				<td title="Actions with master config will automatically apply on slave">Slave for</td>
 				<td>Credentials</td>
 				<td></td>
@@ -266,6 +275,9 @@
 				<td>
 					<label for="typeip"></label><input type="checkbox" id="typeip">
 				</td>
+				<td>
+					<label for="alert"></label><input type="checkbox" id="alert">
+				</td>
 				<td>
 					<select id="slavefor">
 						<option value="0" selected>Not slave</option>

app/templates/ajax/new_server.html

@@ -32,6 +32,13 @@
 						<label for="typeip-{{server.0}}"></label><input type="checkbox" id="typeip-{{server.0}}">
 					{% endif %}
 				</td>
+				<td>
+					{% if server.8 == 1 %}
+						<label for="typeip-{{server.0}}"></label><input type="checkbox" id="typeip-{{server.0}}" checked>
+					{% else %}
+						<label for="typeip-{{server.0}}"></label><input type="checkbox" id="typeip-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option value="0" selected>Not slave</option>

app/templates/base.html

@@ -97,7 +97,7 @@
 					</ul>
 				</nav>
 				<div class="copyright-menu">
-					HAproxy-WI v2.6.3.1
+					HAproxy-WI v2.7
 					<br>
 					<a href="https://www.patreon.com/haproxy_wi" title="Donate" target="_blank" style="color: #fff; margin-left: 30px; color: red;" class="patreon">  Patreon</a>
 				</div>

app/templates/servers.html

@@ -22,6 +22,7 @@
 				<td>IP</td>
 				<td>Enable</td>
 				<td><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
+				<td><span title="Alert if backend change status">Alert(?)</span></td>
 				<td><span title="Actions with master config will automatically apply on slave">Slave for (?)</span></td>
 				<td>Credentials</td>
 				<td></td>
@@ -52,6 +53,13 @@
 						<label for="typeip-{{server.0}}"></label><input type="checkbox" id="typeip-{{server.0}}">
 					{% endif %}
 				</td>
+				<td>
+					{% if server.8 == 1 %}
+						<label for="alert-{{server.0}}"></label><input type="checkbox" id="alert-{{server.0}}" checked>
+					{% else %}
+						<label for="alert-{{server.0}}"></label><input type="checkbox" id="alert-{{server.0}}">
+					{% endif %}
+				</td>
 				<td>
 					<select id="slavefor-{{server.0}}">
 						<option disabled selected>Not slave</option>
@@ -91,7 +99,8 @@
 				<td class="padding10 first-collumn">New hostname</td>
 				<td>IP</td>
 				<td>Enable</td>
-				<td>Virt</td>
+				<td><span title="Vitrual IP, something like VRRP">Virt(?)</span></td>
+				<td><span title="Alert if backend change status">Alert(?)</span></td>
 				<td title="Actions with master config will automatically apply on slave">Slave for</td>
 				<td>Credentials</td>
 				<td></td>
@@ -110,6 +119,9 @@
 				<td>
 					<label for="typeip"></label><input type="checkbox" id="typeip">
 				</td>
+				<td>
+					<label for="alert"></label><input type="checkbox" id="alert">
+				</td>
 				<td>
 					<select id="slavefor">
 						<option disabled selected>Not slave</option>

app/templates/viewstats.html

@@ -14,7 +14,8 @@
 	</select>		
 	<a class="ui-button ui-widget ui-corner-all" id="show" title="Show stats" onclick="{{ onclick }}">Show</a>
 </form>
-<div id="ajax" style="margin-left: 10px;"></div>
+<br />
+<div id="ajax" style="margin-left: 20px; width: 98%"></div>
 <script> 
 	window.onload = showStats();
 	function sleep(ms) {
@@ -27,7 +28,11 @@
 		$('li').css('margin-top', '0');
 		$('table.tbl th.pxname').css('background-color', '#5d9ceb');
 		$('a.px:link').css('color', '#fff');
+		$('h1').css('display', 'none');
 		$('h1').next().css('display', 'none');	  
+		$('h1').next().next().css('display', 'none');	  
+		$('h1').next().next().next().css('display', 'none');	  
+		$('h1').next().next().next().next().css('display', 'none');	  
 		$( "select" ).selectmenu();
 		await sleep(2000);
 	  

app/tools/checker_master.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import subprocess
+import time
+import argparse
+import os, sys
+sys.path.append(os.path.join(sys.path[0], '../'))
+import funct
+import sql
+import signal
+
+class GracefulKiller:
+	kill_now = False
+	def __init__(self):
+		signal.signal(signal.SIGINT, self.exit_gracefully)
+		signal.signal(signal.SIGTERM, self.exit_gracefully)
+	
+	def exit_gracefully(self,signum, frame):
+		self.kill_now = True
+
+def main():
+	servers = sql.select_alert()
+	started_workers = get_worker()
+	servers_list = []
+	
+	for serv in servers:
+		servers_list.append(serv[0])
+			
+	need_kill=list(set(started_workers) - set(servers_list))
+	need_start=list(set(servers_list) - set(started_workers))
+	
+	if need_kill:
+		for serv in need_kill:
+			kill_worker(serv)
+			
+	if need_start:
+		for serv in need_start:
+			start_worker(serv)
+	
+def start_worker(serv):
+	cmd = "tools/checker_worker.py %s &" % serv
+	os.system(cmd)
+	funct.logging("localhost", " Start new worker for: "+serv, alerting=1)
+	
+def kill_worker(serv):
+	cmd = "ps ax |grep 'tools/checker_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv
+	output, stderr = funct.subprocess_execute(cmd)
+	funct.logging("localhost", " Kill worker for: "+serv, alerting=1)
+	if stderr:
+		funct.logging("localhost", stderr, alerting=1)
+
+def kill_all_workers():
+	cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
+	output, stderr = funct.subprocess_execute(cmd)
+	funct.logging("localhost", " Killed all workers", alerting=1)
+	if stderr:
+		funct.logging("localhost", stderr, alerting=1)
+		
+def get_worker():
+	cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $7}'"
+	output, stderr = funct.subprocess_execute(cmd)
+	if stderr:
+		funct.logging("localhost", stderr, alerting=1)
+	return output
+	
+if __name__ == "__main__":
+	funct.logging("localhost", " Master started", alerting=1)
+	killer = GracefulKiller()
+	
+	while True:
+		main()
+		time.sleep(30)
+		
+		if killer.kill_now:
+			break
+			
+	kill_all_workers()
+	funct.logging("localhost", " Master shutdown", alerting=1)

app/tools/checker_worker.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import subprocess
+import time
+import argparse
+import os, sys
+sys.path.append(os.path.join(sys.path[0], '/var/www/haproxy-wi/app'))
+import funct
+import signal
+
+class GracefulKiller:
+	kill_now = False
+	def __init__(self):
+		signal.signal(signal.SIGINT, self.exit_gracefully)
+		signal.signal(signal.SIGTERM, self.exit_gracefully)
+	
+	def exit_gracefully(self,signum, frame):
+		self.kill_now = True
+
+def main(serv, port):
+	port = str(port)
+	firstrun = True
+	currentstat=[]
+	readstats=""
+	killer = GracefulKiller()
+	
+	while True:
+		try:
+			readstats = subprocess.check_output(["echo show stat | nc "+serv+" "+port], shell=True)
+		except:
+			print("Unexpected error:", sys.exc_info())
+			sys.exit()
+			
+		vips = readstats.splitlines()
+		
+		for i in range(0,len(vips)):
+			if "UP" in str(vips[i]):		 			
+				currentstat.append("UP")
+			elif "DOWN" in str(vips[i]):
+				currentstat.append("DOWN")
+			elif "MAINT" in str(vips[i]):
+				currentstat.append("MAINT")
+			else:
+				currentstat.append("none")
+
+			if firstrun == False:
+				if (currentstat[i] != oldstat[i] and currentstat[i]!="none") and ("FRONTEND" not in str(vips[i]) and "BACKEND" not in str(vips[i])):
+					servername= str(vips[i])
+					servername=servername.split(",")
+					realserver = servername[0]
+					alert=realserver[2:]+ " has changed status and is now "+ currentstat[i] + " at " + serv 
+					funct.telegram_send_mess(str(alert))
+		firstrun=False
+		oldstat=[]
+		oldstat=currentstat
+		currentstat=[]
+		time.sleep(30)	
+		
+		
+		if killer.kill_now:
+			break
+	
+	funct.logging("localhost", " Worker shutdown for: "+serv, alerting=1)
+
+if __name__ == "__main__":
+	parser = argparse.ArgumentParser(description='Check HAProxy servers state.', prog='check_haproxy.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+	
+	parser.add_argument('IP', help='Start check HAProxy server state at this ip', nargs='?', type=str)
+	parser.add_argument('--port', help='Start check HAProxy server state at this port', nargs='?', default=1999, type=int)
+					
+	args = parser.parse_args()
+	if args.IP is None:
+		parser.print_help()
+		import sys
+		sys.exit()
+	else: 
+		try:
+			main(args.IP, args.port)
+		except KeyboardInterrupt:
+			pass

inc/script.js

@@ -3,6 +3,19 @@ var cur_url = window.location.href.split('/').pop();
 cur_url = cur_url.split('?');
 var intervalId;
 
+jQuery.expr[':'].regex = function(elem, index, match) {
+    var matchParams = match[3].split(','),
+        validLabels = /^(data|css):/,
+        attr = {
+            method: matchParams[0].match(validLabels) ? 
+                        matchParams[0].split(':')[0] : 'attr',
+            property: matchParams.shift().replace(validLabels,'')
+        },
+        regexFlags = 'ig',
+        regex = new RegExp(matchParams.join('').replace(/^\s+|\s+$/g,''), regexFlags);
+    return regex.test(jQuery(elem)[attr.method](attr.property));
+}
+
 function autoRefreshStyle(autoRefresh) {
 	var margin;
 	if (cur_url[0] == "overview.py") {

inc/users.js

@@ -180,12 +180,16 @@ $( function() {
 		$('.alert-danger').remove();	
 		var typeip = 0;
 		var enable = 0;
+		var alert_en = 0;
 		if ($('#typeip').is(':checked')) {
 			typeip = '1';
 		}
 		if ($('#enable').is(':checked')) {
 			enable = '1';
 		}
+		if ($('#alert').is(':checked')) {
+			var alert_en = 0;
+		}
 		$.ajax( {
 			url: "sql.py",
 			data: {
@@ -196,6 +200,7 @@ $( function() {
 				enable: enable,
 				slave: $('#slavefor' ).val(),
 				cred: $('#credentials').val(),
+				alert_en: alert_en
 			},
 			type: "GET",
 			success: function( data ) {
@@ -545,12 +550,16 @@ function updateServer(id) {
 	$('.alert-danger').remove();
 	var typeip = 0;
 	var enable = 0;
+	var alert_en = 0;
 	if ($('#typeip-'+id).is(':checked')) {
 		typeip = '1';
 	}
 	if ($('#enable-'+id).is(':checked')) {
 		enable = '1';
 	}
+	if ($('#alert-'+id).is(':checked')) {
+		alert_en = '1';
+	}
 	$.ajax( {
 		url: "sql.py",
 		data: {
@@ -561,7 +570,8 @@ function updateServer(id) {
 			enable: enable,
 			slave: $('#slavefor-'+id+' option:selected' ).val(),
 			cred: $('#credentials-'+id+' option:selected').val(),
-			id: id
+			id: id,
+			alert_en: alert_en
 		},
 		type: "GET",
 		success: function( data ) {

install.sh

@@ -151,6 +151,28 @@ cat << EOF > $HAPROXY_WI_VHOST_CONF
 EOF
 fi 
 
+cat << EOF > /etc/systemd/system/multi-user.target.wants/checker_haproxy.service
+[Unit]
+Description=Haproxy backends state checker
+After=syslog.target network.target
+
+[Service]
+Type=simple
+WorkingDirectory=/var/www/$HOME_HAPROXY_WI
+ExecStart=/var/www/$HOME_HAPROXY_WI/tools/checker_master.py
+
+RestartSec=2s
+Restart=on-failure
+TimeoutStopSec=1s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload      
+systemctl start checker_haproxy.service
+systemctl enable checker_haproxy.service
+
 if hash apt-get 2>/dev/null; then
 	sed -i 's|/var/log/httpd/|/var/log/apache2/|g' $HAPROXY_WI_VHOST_CONF
 	cd /etc/apache2/mods-enabled
@@ -321,6 +343,7 @@ echo "################################"
 mkdir /var/www/$HOME_HAPROXY_WI/app/certs
 mkdir /var/www/$HOME_HAPROXY_WI/keys
 chmod +x /var/www/$HOME_HAPROXY_WI/app/*.py
+chmod +x /var/www/$HOME_HAPROXY_WI/app/tools/*.py
 rm -f /var/www/$HOME_HAPROXY_WI/log/config_edit.log
 cd /var/www/$HOME_HAPROXY_WI/app
 ./update_db.py

requirements.txt

@@ -3,7 +3,7 @@ paramiko==2.4.1
 pytz==2017.3
 requests==2.18.4
 requests_toolbelt==0.8.0
-telegram==0.0.1
+pyTelegramBotAPI==3.6.3 
 dump==0.0.3
 networkx==2.1
 numpy==1.14.0

update.sh

@@ -8,8 +8,32 @@ git pull  https://github.com/Aidaho12/haproxy-wi.git
 mkdir keys
 mkdir app/certs
 chmod +x app/*py
+chmod +x app/tools/*py
 chown -R apache:apache *
 
+
+cat << EOF > /etc/systemd/system/multi-user.target.wants/checker_haproxy.service
+[Unit]
+Description=Haproxy backends state checker
+After=syslog.target network.target
+
+[Service]
+Type=simple
+WorkingDirectory=/var/www/haproxy-wi/app/
+ExecStart=/var/www/haproxy-wi/app/tools/checker_master.py
+
+RestartSec=2s
+Restart=on-failure
+TimeoutStopSec=1s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload      
+systemctl start checker_haproxy.service
+systemctl enable checker_haproxy.service
+
 cd app/
 ./update_db.py