From 710575dbab29f8f0babfc52e215bdd964c8fd729 Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Wed, 2 Sep 2020 10:26:36 +0600 Subject: [PATCH] v4.4.3.0 Changelog; https://haproxy-wi.org/changelog.py#4_4_3 --- app/create_db.py | 35 +++- app/options.py | 54 +++++- app/smon.py | 4 +- app/templates/ajax/stick_table.html | 6 +- app/templates/ajax/stick_tables.html | 6 +- app/templates/viewstats.html | 80 ++++++++- app/templates/waf.html | 180 +++++++++++-------- app/tools/metrics_master.py | 123 ------------- app/tools/metrics_waf_worker.py | 64 ------- app/tools/metrics_worker.py | 68 ------- config_other/logrotate/metrics | 9 - config_other/syslog/metrics.conf | 2 - config_other/systemd/metrics_haproxy.service | 19 -- inc/runtimeapi.js | 59 ++++-- inc/script.js | 1 - inc/waf.js | 19 +- 16 files changed, 328 insertions(+), 401 deletions(-) delete mode 100644 app/tools/metrics_master.py delete mode 100644 app/tools/metrics_waf_worker.py delete mode 100644 app/tools/metrics_worker.py delete mode 100644 config_other/logrotate/metrics delete mode 100644 config_other/syslog/metrics.conf delete mode 100644 config_other/systemd/metrics_haproxy.service diff --git a/app/create_db.py b/app/create_db.py index 6591c4e2..5027048e 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -534,12 +534,12 @@ def update_db_v_4_3_2(**kwargs): except sqltool.Error as e: if kwargs.get('silent') != 1: if e.args[0] == 'columns param, group are not unique' or e == " 1060 (42S21): columns param, group are not unique ": - print('DB was update to 4.3.2') + print('Updating... go to version 4.4.0') else: print("An error occurred:", e) return False else: - print("DB was update to 4.3.2") + print("Updating... go to version 4.4.0") return True cur.close() con.close() @@ -630,11 +630,38 @@ def update_db_v_4_4_2_1(**kwargs): return True cur.close() con.close() + + +def update_db_v_4_5(**kwargs): + con, cur = get_cur() + sql = """CREATE TABLE IF NOT EXISTS `alerts` (`id` INTEGER NOT NULL, + `message` varchar(64), + `level` varchar(64), + `ip` varchar(64), + `port` INTEGER, + `user_group` INTEGER default 1, + `service` varchar(64), + `date` DATETIME default '0000-00-00 00:00:00', + PRIMARY KEY(`id`) ); """ + try: + cur.execute(sql) + con.commit() + except sqltool.Error as e: + if kwargs.get('silent') != 1: + if e.args[0] == 'duplicate column name: version' or e == "1060 (42S21): Duplicate column name 'version' ": + print('Updating... go to version 4.5.1') + else: + print("Updating... go to version to 4.5.1") + return False + else: + return True + cur.close() + con.close() def update_ver(**kwargs): con, cur = get_cur() - sql = """update version set version = '4.4.2.0'; """ + sql = """update version set version = '4.4.3.0'; """ try: cur.execute(sql) con.commit() @@ -666,6 +693,7 @@ def update_all(): update_db_v_4_4() update_db_v_4_4_2() update_db_v_4_4_2_1() + update_db_v_4_5() update_ver() @@ -691,6 +719,7 @@ def update_all_silent(): update_db_v_4_4(silent=1) update_db_v_4_4_2(silent=1) update_db_v_4_4_2_1(silent=1) + update_db_v_4_5(silent=1) update_ver() diff --git a/app/options.py b/app/options.py index ecc827a0..7cbf606c 100644 --- a/app/options.py +++ b/app/options.py @@ -25,9 +25,10 @@ if act == "checkrestart": sys.exit() -if not sql.check_token_exists(form.getvalue("token")): - print('error: Your token has been expired') - sys.exit() +if form.getvalue('alert_consumer') is None: + if not sql.check_token_exists(form.getvalue("token")): + print('error: Your token has been expired') + sys.exit() if form.getvalue('getcerts') is not None and serv is not None: @@ -244,6 +245,16 @@ if form.getvalue('ip_for_delete') is not None: print('error: ' + stderr[0]) +if form.getvalue('table_for_clear') is not None: + haproxy_sock_port = sql.get_setting('haproxy_sock_port') + table = form.getvalue('table_for_clear') + + cmd='echo "clear table %s " |nc %s %s' % (table, serv, haproxy_sock_port) + output, stderr = funct.subprocess_execute(cmd) + if stderr[0] != '': + print('error: ' + stderr[0]) + + if form.getvalue('list_serv_select') is not None: haproxy_sock_port = sql.get_setting('haproxy_sock_port') cmd='echo "show acl"|nc %s %s |grep "loaded from" |awk \'{print $1,$2}\'' % (serv, haproxy_sock_port) @@ -2023,4 +2034,39 @@ if form.getvalue('showBytes') is not None: env = Environment(loader=FileSystemLoader('templates'), autoescape=True) template = env.get_template('ajax/bin_bout.html') template = template.render(bin_bout=bin_bout,serv=serv) - print(template) \ No newline at end of file + print(template) + + +if form.getvalue('alert_consumer'): + try: + user_group = funct.get_user_group(id=1) + if funct.check_user_group(): + message = sql.select_alerts(user_group) + for m in message: + print(m[0]+ ': '+ m[1] +' date: '+m[2]+';') + except: + pass + + +if form.getvalue('waf_rule_id'): + enable = form.getvalue('waf_en') + rule_id = form.getvalue('waf_rule_id') + haproxy_path = sql.get_setting('haproxy_dir') + rule_file = sql.select_waf_rule_by_id(rule_id) + conf_file_path = haproxy_path + 'waf/modsecurity.conf' + rule_file_path = 'Include ' + haproxy_path + '/waf/rules/' + rule_file + + if enable == '0': + cmd = ["sudo sed -i 's!"+rule_file_path+"!#"+rule_file_path+"!' "+conf_file_path] + en_for_log = 'disable' + else: + cmd = ["sudo sed -i 's!#"+rule_file_path+"!"+rule_file_path+"!' "+conf_file_path] + en_for_log = 'enable' + + try: + funct.logging('WAF', ' Has been '+en_for_log+' WAF rule: '+rule_file+' for the server '+serv, haproxywi=1, login=1) + except: + pass + + print(funct.ssh_command(serv, cmd)) + sql.update_enable_waf_rules(rule_id, serv, enable) \ No newline at end of file diff --git a/app/smon.py b/app/smon.py index 13dbaa75..2324cb98 100644 --- a/app/smon.py +++ b/app/smon.py @@ -25,13 +25,15 @@ if action == 'add': smon = sql.select_smon(user_group,action='add') funct.page_for_admin(level=2) title = "SMON Admin" + autorefresh = 0 else: smon = sql.smon_list(user_group) title = "SMON Dashboard" + autorefresh = 1 template = template.render(h2 = 1, title = title, - autoreœfresh = 1, + autorefresh = autorefresh, role = role, user = user, group = user_group, diff --git a/app/templates/ajax/stick_table.html b/app/templates/ajax/stick_table.html index 468e6537..7401d1d5 100644 --- a/app/templates/ajax/stick_table.html +++ b/app/templates/ajax/stick_table.html @@ -33,9 +33,9 @@ Used: {{tables_head.3}} - - - + + + diff --git a/app/templates/ajax/stick_tables.html b/app/templates/ajax/stick_tables.html index 9d14f8c5..131c3d64 100644 --- a/app/templates/ajax/stick_tables.html +++ b/app/templates/ajax/stick_tables.html @@ -34,9 +34,9 @@ - - - +
Used: {{t.0.3}} + +
{%- if t.1.0 != '' -%} diff --git a/app/templates/viewstats.html b/app/templates/viewstats.html index d945b09f..23410a54 100644 --- a/app/templates/viewstats.html +++ b/app/templates/viewstats.html @@ -1,9 +1,13 @@ {% extends "base.html" %} -{% block content %} +{% block content %} +{% from 'include/input_macros.html' import input, checkbox, select %}
{% if selects|length == 0 %} {% include 'include/getstarted.html' %} {% else %} +
Show +
+
Filter:
+ {{ checkbox('stats_active', title='Show active servers', desc='Active', checked='checked') }} + {{ checkbox('stats_drain', title='Show drain servers', desc='Drain', checked='checked') }} + {{ checkbox('stats_maintain', title='Show maintain servers', desc='Maintain', checked='checked') }} + {{ checkbox('stats_down', title='Show DOWN servers', desc='DOWN', checked='checked') }} + {{ checkbox('stats_not_checked', title='Show not checked servers', desc='Not checked', checked='checked') }} + {{ checkbox('stats_backup', title='Show backup servers', desc='Backup', checked='checked') }} + {{ checkbox('stats_frontends', title='Show frontends servers', desc='Frontends', checked='checked') }} + {{ checkbox('stats_backends', title='Show backends servers', desc='Backends', checked='checked') }} +
-
+
Please choose a server -
+
+ {% endif %} diff --git a/app/templates/waf.html b/app/templates/waf.html index 10f67c49..5e098d06 100644 --- a/app/templates/waf.html +++ b/app/templates/waf.html @@ -1,79 +1,113 @@ {% extends "base.html" %} -{% block content %} - - -{% if servers_all|length == 0 %} - {% include 'include/getstarted.html' %} -{% else %} - - - - - - - - - {% for s in servers_all %} - - {% endfor %} -
- Server - - Action - - WAF mode - - Metrics - - - - -
- - - - - +{% block content %} -
- -
-{% for s in servers %} -
- -
-{% endfor %} - - -
- You can read the description and watch a video about WAF here -
+ + + {% if servers_all|length == 0 %} + {% include 'include/getstarted.html' %} + {% else %} + + + + + + + + + + {% for s in servers_all %} + + {% endfor %} +
+ Server + + Actions + + WAF mode + + Metrics + + Manage rules + + + + +
+ + + + + +
+ +
+ {% for s in servers %} +
+ +
+ {% endfor %} + + +
+ You can read the description and watch a video about WAF here +
+ {% endif %} {% endif %} {% endblock %} \ No newline at end of file diff --git a/app/tools/metrics_master.py b/app/tools/metrics_master.py deleted file mode 100644 index 6b296d2b..00000000 --- a/app/tools/metrics_master.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import time -import argparse -import os, sys -sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) -sys.path.append(os.path.join(sys.path[0], os.getcwd())) -import funct -import sql -import signal - -class GracefulKiller: - kill_now = False - def __init__(self): - signal.signal(signal.SIGINT, self.exit_gracefully) - signal.signal(signal.SIGTERM, self.exit_gracefully) - - def exit_gracefully(self,signum, frame): - self.kill_now = True - -def main(): - sql.delete_mentrics() - sql.delete_waf_mentrics() - servers = sql.select_servers_metrics_for_master() - started_workers = get_worker() - servers_list = [] - - for serv in servers: - servers_list.append(serv[0]) - - need_kill=list(set(started_workers) - set(servers_list)) - need_start=list(set(servers_list) - set(started_workers)) - - if need_kill: - for serv in need_kill: - kill_worker(serv) - - if need_start: - for serv in need_start: - start_worker(serv) - - try: - waf_servers = sql.select_all_waf_servers() - waf_started_workers = get_waf_worker() - waf_servers_list = [] - - for serv in waf_servers: - waf_servers_list.append(serv[0]) - - waf_need_kill=list(set(waf_started_workers) - set(waf_servers_list)) - waf_need_start=list(set(waf_servers_list) - set(waf_started_workers)) - - if waf_need_kill: - for serv in waf_need_kill: - kill_waf_worker(serv) - - if waf_need_start: - for serv in waf_need_start: - start_waf_worker(serv) - except Exception as e: - funct.logging("localhost", 'Problems with WAF worker metrics '+e, metrics=1) - pass - -def start_worker(serv): - port = sql.get_setting('haproxy_sock_port') - cmd = "tools/metrics_worker.py %s --port %s &" % (serv, port) - os.system(cmd) - funct.logging("localhost", " Master started new metrics worker for: "+serv, metrics=1) - -def kill_worker(serv): - cmd = "ps ax |grep 'tools/metrics_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv - output, stderr = funct.subprocess_execute(cmd) - funct.logging("localhost", " Master killed metrics worker for: "+serv, metrics=1) - if stderr: - funct.logging("localhost", stderr, metrics=1) - -def start_waf_worker(serv): - port = sql.get_setting('haproxy_sock_port') - cmd = "tools/metrics_waf_worker.py %s --port %s &" % (serv, port) - os.system(cmd) - funct.logging("localhost", " Master started new WAF metrics worker for: "+serv, metrics=1) - -def kill_waf_worker(serv): - cmd = "ps ax |grep 'tools/metrics_waf_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv - output, stderr = funct.subprocess_execute(cmd) - funct.logging("localhost", " Master killed WAF metrics worker for: "+serv, metrics=1) - if stderr: - funct.logging("localhost", stderr, metrics=1) - -def kill_all_workers(): - cmd = "ps ax |grep -e 'tools/metrics_worker.py\|tools/metrics_waf_worker.py' |grep -v grep |awk '{print $1}' |xargs kill" - output, stderr = funct.subprocess_execute(cmd) - funct.logging("localhost", " Master killing all metrics workers", metrics=1) - if stderr: - funct.logging("localhost", stderr, metrics=1) - -def get_worker(): - cmd = "ps ax |grep 'tools/metrics_worker.py' |grep -v grep |awk '{print $7}'" - output, stderr = funct.subprocess_execute(cmd) - if stderr: - funct.logging("localhost", stderr, metrics=1) - return output - -def get_waf_worker(): - cmd = "ps ax |grep 'tools/metrics_waf_worker.py' |grep -v grep |awk '{print $7}'" - output, stderr = funct.subprocess_execute(cmd) - if stderr: - funct.logging("localhost", stderr, metrics=1) - return output - -if __name__ == "__main__": - funct.logging("localhost", " Metrics master started", metrics=1) - killer = GracefulKiller() - - while True: - main() - time.sleep(20) - - if killer.kill_now: - break - - kill_all_workers() - funct.logging("localhost", " Master shutdown", metrics=1) \ No newline at end of file diff --git a/app/tools/metrics_waf_worker.py b/app/tools/metrics_waf_worker.py deleted file mode 100644 index fd8c8b06..00000000 --- a/app/tools/metrics_waf_worker.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -from subprocess import check_output, CalledProcessError -import time -import argparse -import os, sys -sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) -sys.path.append(os.path.join(sys.path[0], os.getcwd())) -import sql -import signal - -class GracefulKiller: - kill_now = False - def __init__(self): - signal.signal(signal.SIGINT, self.exit_gracefully) - signal.signal(signal.SIGTERM, self.exit_gracefully) - - def exit_gracefully(self,signum, frame): - self.kill_now = True - -def main(serv, port): - port = str(port) - firstrun = True - readstats = "" - killer = GracefulKiller() - - while True: - try: - cmd = "echo 'show stat' |nc "+serv+" "+port+" | cut -d ',' -f 1-2,5 |grep waf |grep BACKEND |awk -F',' '{print $3}'" - readstats = subprocess.check_output([cmd], shell=True) - except CalledProcessError as e: - print("Command error") - except OSError as e: - print(e) - sys.exit() - readstats = readstats.decode(encoding='UTF-8') - metric = readstats.splitlines() - - try: - sql.insert_waf_mentrics(serv, metric[0]) - except: - pass - - time.sleep(30) - - if killer.kill_now: - break - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Metrics HAProxy service.', prog='metrics_worker.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('IP', help='Start get metrics from HAProxy service at this ip', nargs='?', type=str) - parser.add_argument('--port', help='Start get metrics from HAProxy service at this port', nargs='?', default=1999, type=int) - - args = parser.parse_args() - if args.IP is None: - parser.print_help() - import sys - sys.exit() - else: - try: - main(args.IP, args.port) - except KeyboardInterrupt: - pass \ No newline at end of file diff --git a/app/tools/metrics_worker.py b/app/tools/metrics_worker.py deleted file mode 100644 index b3754b94..00000000 --- a/app/tools/metrics_worker.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -from subprocess import check_output, CalledProcessError -import time -import argparse -import os, sys -sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) -sys.path.append(os.path.join(sys.path[0], os.getcwd())) -import sql -import signal - -class GracefulKiller: - kill_now = False - def __init__(self): - signal.signal(signal.SIGINT, self.exit_gracefully) - signal.signal(signal.SIGTERM, self.exit_gracefully) - - def exit_gracefully(self,signum, frame): - self.kill_now = True - -def main(serv, port): - port = str(port) - firstrun = True - readstats = "" - killer = GracefulKiller() - - while True: - try: - cmd = "echo show info | nc "+serv+" "+port+" |grep -e 'CurrConns\|CurrSslConns\|MaxSessRate:\|SessRate:'|awk '{print $2}'" - readstats = subprocess.check_output([cmd], shell=True) - except CalledProcessError as e: - print("Command error") - except OSError as e: - print(e) - sys.exit() - - try: - readstats = readstats.decode(encoding='UTF-8') - metric = readstats.splitlines() - metrics = [] - for i in range(0,len(metric)): - metrics.append(metric[i]) - - sql.insert_mentrics(serv, metrics[0], metrics[1], metrics[2], metrics[3]) - except Exception as e: - print(str(e)) - - time.sleep(30) - - if killer.kill_now: - break - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Metrics HAProxy service.', prog='metrics_worker.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('IP', help='Start get metrics from HAProxy service at this ip', nargs='?', type=str) - parser.add_argument('--port', help='Start get metrics from HAProxy service at this port', nargs='?', default=1999, type=int) - - args = parser.parse_args() - if args.IP is None: - parser.print_help() - import sys - sys.exit() - else: - try: - main(args.IP, args.port) - except KeyboardInterrupt: - pass \ No newline at end of file diff --git a/config_other/logrotate/metrics b/config_other/logrotate/metrics deleted file mode 100644 index 7f227ee9..00000000 --- a/config_other/logrotate/metrics +++ /dev/null @@ -1,9 +0,0 @@ -/var/www/haproxy-wi/log/metrics-error.log { - daily - rotate 10 - missingok - notifempty - create 0644 apache apache - dateext - sharedscripts -} diff --git a/config_other/syslog/metrics.conf b/config_other/syslog/metrics.conf deleted file mode 100644 index 1b9b308a..00000000 --- a/config_other/syslog/metrics.conf +++ /dev/null @@ -1,2 +0,0 @@ -if $programname startswith 'metrics' then /var/www/haproxy-wi/log/metrics-error.log -& stop diff --git a/config_other/systemd/metrics_haproxy.service b/config_other/systemd/metrics_haproxy.service deleted file mode 100644 index 28c11637..00000000 --- a/config_other/systemd/metrics_haproxy.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Haproxy metrics -After=syslog.target network.target - -[Service] -Type=simple -WorkingDirectory=/var/www/haproxy-wi/app/ -ExecStart=/var/www/haproxy-wi/app/tools/metrics_master.py - -StandardOutput=syslog -StandardError=syslog -SyslogIdentifier=metrics - -RestartSec=2s -Restart=on-failure -TimeoutStopSec=1s - -[Install] -WantedBy=multi-user.target diff --git a/inc/runtimeapi.js b/inc/runtimeapi.js index 4089b799..cf22be29 100644 --- a/inc/runtimeapi.js +++ b/inc/runtimeapi.js @@ -222,25 +222,7 @@ $( function() { } ); }); $('#runtimeapitable').submit(function() { - $.ajax( { - url: "options.py", - data: { - serv: $('#table_serv_select').val(), - table_select: $('#table_select').val(), - token: $('#token').val() - }, - type: "POST", - success: function( data ) { - if (data.indexOf('error:') != '-1') { - toastr.error(data); - } else { - $("#ajaxtable").html(data); - $( "input[type=submit], button" ).button(); - $.getScript("/inc/fontawesome.min.js"); - FontAwesomeConfig = { searchPseudoElements: true, observeMutations: false }; - } - } - } ); + getTable(); return false; }); $('#runtimeapilist').submit(function() { @@ -309,6 +291,45 @@ function deleteTableEntry(id, table, ip) { } } ); } +function clearTable(table) { + $.ajax( { + url: "options.py", + data: { + serv: $('#table_serv_select').val(), + table_for_clear: table, + token: $('#token').val() + }, + type: "POST", + success: function( data ) { + if (data.indexOf('error: ') != '-1') { + toastr.error(data); + } else { + getTable(); + } + } + } ); +} +function getTable() { + $.ajax( { + url: "options.py", + data: { + serv: $('#table_serv_select').val(), + table_select: $('#table_select').val(), + token: $('#token').val() + }, + type: "POST", + success: function( data ) { + if (data.indexOf('error:') != '-1') { + toastr.error(data); + } else { + $("#ajaxtable").html(data); + $( "input[type=submit], button" ).button(); + $.getScript("/inc/fontawesome.min.js"); + FontAwesomeConfig = { searchPseudoElements: true, observeMutations: false }; + } + } + } ); +} function getList() { $.ajax( { url: "options.py", diff --git a/inc/script.js b/inc/script.js index c37faef6..a89f859b 100644 --- a/inc/script.js +++ b/inc/script.js @@ -823,7 +823,6 @@ $( function() { success: function( data ) { if (data.indexOf('ok') != '-1') { window.location.replace(ref); - console.log(data) } else if (data.indexOf('disabled') != '-1') { $('.alert').show(); $('.alert').html(data); diff --git a/inc/waf.js b/inc/waf.js index c3fc5ebf..626949b5 100644 --- a/inc/waf.js +++ b/inc/waf.js @@ -109,20 +109,29 @@ function waf_rules_en(id) { if ($('#rule_id-'+id).is(':checked')) { enable = '1'; } + var serv = findGetParameter('serv') + console.log(serv) + console.log(id) + console.log(enable) $.ajax( { url: "options.py", data: { waf_rule_id: id, waf_en: enable, + serv: serv, token: $('#token').val() }, type: "POST", success: function( data ) { - toastr.info('Do not forget restart WAF service'); - $( '#rule-'+id ).addClass( "update", 1000 ); - setTimeout(function() { - $( '#rule-'+id ).removeClass( "update" ); - }, 2500 ); + if (data.indexOf('sed:') != '-1' || data.indexOf('error: ') != '-1' ) { + toastr.error(data); + } else { + toastr.info('Do not forget restart WAF service'); + $('#rule-' + id).addClass("update", 1000); + setTimeout(function () { + $('#rule-' + id).removeClass("update"); + }, 2500); + } } } ); } \ No newline at end of file