From 769df0f6c4671e744e2387733f8623ddc7ffddfe Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Mon, 16 Nov 2020 18:31:52 +0600 Subject: [PATCH] v4.5.4.0 Changelog: https://haproxy-wi.org/changelog.py#4_5_4 --- app/create_db.py | 46 +++++++++++----------- app/hapservers.py | 34 ++++++++-------- app/options.py | 54 +++++++++++++------------- app/overview.py | 64 +++++++++++++++--------------- app/sql.py | 73 ++++++++++++++++++++++++++--------- app/templates/base.html | 1 + app/templates/hapservers.html | 46 +++++++++++----------- inc/add.js | 49 ++++++++++++----------- inc/awesome.css | 2 +- inc/overview.js | 18 +++++++-- inc/script.js | 9 +++++ 11 files changed, 228 insertions(+), 168 deletions(-) diff --git a/app/create_db.py b/app/create_db.py index 99b4c667..4b4e4cac 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -188,7 +188,6 @@ def update_db_v_31(**kwargs): sql.append("INSERT INTO settings (param, value, section, `desc`) values('server_state_file', '/etc/haproxy/haproxy.state', 'haproxy', 'Path to HAProxy state file');") sql.append("INSERT INTO settings (param, value, section, `desc`) values('haproxy_sock', '/var/run/haproxy.sock', 'haproxy', 'Path to HAProxy sock file');") sql.append("INSERT INTO settings (param, value, section, `desc`) values('haproxy_sock_port', '1999', 'haproxy', 'HAProxy sock port');") - sql.append("INSERT INTO settings (param, value, section, `desc`) values('firewall_enable', '0', 'haproxy', 'If enable this option Haproxy-wi will be configure firewalld based on config port');") sql.append("INSERT INTO settings (param, value, section, `desc`) values('apache_log_path', '/var/log/httpd/', 'logs', 'Path to Apache logs');") sql.append("INSERT INTO settings (param, value, section, `desc`) values('ldap_enable', '0', 'ldap', 'If 1 ldap enabled');") sql.append("INSERT INTO settings (param, value, section, `desc`) values('ldap_server', '', 'ldap', 'IP address ldap server');") @@ -432,26 +431,7 @@ def update_db_v_42(**kwargs): cur.close() con.close() - - -def update_db_v_4_2_3(**kwargs): - con, cur = get_cur() - sql = """ - update settings set section = 'main' where param = 'firewall_enable'; - """ - try: - cur.execute(sql) - con.commit() - except sqltool.Error as e: - if kwargs.get('silent') != 1: - if e.args[0] == 'duplicate column name: haproxy' or e == " 1060 (42S21): Duplicate column name 'haproxy' ": - print('Updating... go to version 4.3.0') - else: - print("An error occurred:", e) - cur.close() - con.close() - def update_db_v_4_3(**kwargs): con, cur = get_cur() @@ -705,11 +685,31 @@ def update_db_v_4_5_1(**kwargs): print('DB was update to 4.5.0') cur.close() con.close() + + +def update_db_v_4_5_4(**kwargs): + con, cur = get_cur() + sql = list() + sql.append("ALTER TABLE `servers` ADD COLUMN `nginx_active` INTEGER NOT NULL DEFAULT 0;") + sql.append("ALTER TABLE `servers` ADD COLUMN `firewall_enable` INTEGER NOT NULL DEFAULT 0;") + sql.append("delete from settings where param = 'firewall_enable';") + for i in sql: + try: + cur.execute(i) + con.commit() + except sqltool.Error as e: + pass + else: + if kwargs.get('silent') != 1: + print('DB was update to 4.5.4') + + cur.close() + con.close() def update_ver(**kwargs): con, cur = get_cur() - sql = """update version set version = '4.5.3.0'; """ + sql = """update version set version = '4.5.4.0'; """ try: cur.execute(sql) con.commit() @@ -733,7 +733,6 @@ def update_all(): update_db_v_4() update_db_v_41() update_db_v_42() - update_db_v_4_2_3() update_db_v_4_3() update_db_v_4_3_0() update_db_v_4_3_1() @@ -744,6 +743,7 @@ def update_all(): update_db_v_4_3_2_1() update_db_v_4_5() update_db_v_4_5_1() + update_db_v_4_5_4() update_ver() @@ -761,7 +761,6 @@ def update_all_silent(): update_db_v_4(silent=1) update_db_v_41(silent=1) update_db_v_42(silent=1) - update_db_v_4_2_3(silent=1) update_db_v_4_3(silent=1) update_db_v_4_3_0(silent=1) update_db_v_4_3_1(silent=1) @@ -772,6 +771,7 @@ def update_all_silent(): update_db_v_4_3_2_1(silent=1) update_db_v_4_5(silent=1) update_db_v_4_5_1(silent=1) + update_db_v_4_5_4(silent=1) update_ver() diff --git a/app/hapservers.py b/app/hapservers.py index 4aa977ac..3a80efc3 100644 --- a/app/hapservers.py +++ b/app/hapservers.py @@ -22,11 +22,11 @@ form = funct.form serv = form.getvalue('serv') service = form.getvalue('service') autorefresh = 0 +cmd = "ps ax |grep -e 'keep_alive.py' |grep -v grep |wc -l" +keep_alive, stderr = funct.subprocess_execute(cmd) if service == 'nginx': title = "Nginx servers overview" - keep_alive = '' - stderr = '' servers = sql.get_dick_permit(virt=1, nginx=1) service = 'nginx' if serv: @@ -35,8 +35,6 @@ if service == 'nginx': autorefresh = 1 else: title = "HAProxy servers overview" - cmd = "ps ax |grep -e 'keep_alive.py' |grep -v grep |wc -l" - keep_alive, stderr = funct.subprocess_execute(cmd) service = 'haproxy' if serv: if funct.check_is_server_in_group(serv): @@ -45,18 +43,18 @@ else: else: servers = sql.get_dick_permit(virt=1, haproxy=1) - services_name = {'checker_haproxy': 'Master backends checker service', - 'keep_alive': 'Auto start service', - 'metrics_haproxy': 'Master metrics service'} - for s, v in services_name.items(): - if s != 'keep_alive': - service_name = s.split('_')[0] - else: - service_name = s - cmd = "rpm --query haproxy-wi-" + service_name + "-* |awk -F\"" + service_name +"\" '{print $2}' |awk -F\".noa\" '{print $1}' |sed 's/-//1' |sed 's/-/./'" - service_ver, stderr = funct.subprocess_execute(cmd) +services_name = {'checker_haproxy': 'Master backends checker service', + 'keep_alive': 'Auto start service', + 'metrics_haproxy': 'Master metrics service'} +for s, v in services_name.items(): + if s != 'keep_alive': + service_name = s.split('_')[0] + else: + service_name = s + cmd = "rpm --query haproxy-wi-" + service_name + "-* |awk -F\"" + service_name + "\" '{print $2}' |awk -F\".noa\" '{print $1}' |sed 's/-//1' |sed 's/-/./'" + service_ver, stderr = funct.subprocess_execute(cmd) - services.append([s, service_ver[0]]) + services.append([s, service_ver[0]]) haproxy_sock_port = sql.get_setting('haproxy_sock_port') servers_with_status1 = [] @@ -87,8 +85,10 @@ for s in servers: else: out1 = False servers_with_status.append(out1) - - servers_with_status.append(s[12]) + if service == 'haproxy': + servers_with_status.append(s[12]) + else: + servers_with_status.append(s[17]) servers_with_status.append(sql.is_master(s[2])) servers_with_status.append(sql.select_servers(server=s[2])) diff --git a/app/options.py b/app/options.py index 13b221c7..b2fb8cb6 100644 --- a/app/options.py +++ b/app/options.py @@ -43,9 +43,9 @@ if form.getvalue('checkSshConnect') is not None and serv is not None: print(funct.ssh_command(serv, ["ls -1t"])) if form.getvalue('getcert') is not None and serv is not None: - id = form.getvalue('getcert') + cert_id = form.getvalue('getcert') cert_path = sql.get_setting('cert_path') - commands = ["cat " + cert_path + "/" + id] + commands = ["cat " + cert_path + "/" + cert_id] try: funct.ssh_command(serv, commands, ip="1") except Exception as e: @@ -206,12 +206,12 @@ if form.getvalue('table_select') is not None: table = [] for t in tables.split(','): if t != '': - id = [] + table_id = [] tables_head = [] tables_head1, table1 = funct.get_stick_table(t) - id.append(tables_head1) - id.append(table1) - table.append(id) + table_id.append(tables_head1) + table_id.append(table1) + table.append(table_id) template = template.render(table=table) else: @@ -561,8 +561,7 @@ if act == "overviewServers": async def async_get_overviewServers(serv1, serv2, service): if service == 'haproxy': - cmd = 'echo "show info" |nc %s %s -w 1|grep -e "Ver\|CurrConns\|Maxco\|MB\|Uptime:"' % ( - serv2, sql.get_setting('haproxy_sock_port')) + cmd = 'echo "show info" |nc %s %s -w 1|grep -e "Ver\|CurrConns\|Maxco\|MB\|Uptime:"' % (serv2, sql.get_setting('haproxy_sock_port')) out = funct.subprocess_execute(cmd) out1 = "" @@ -1061,7 +1060,7 @@ if form.getvalue('masteradd'): print(output) break else: - print('success: Master VRRP address was added
') + print('success: Master VRRP address has been added
') ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = funct.return_ssh_keys_path(slave) @@ -1094,7 +1093,7 @@ if form.getvalue('masteradd'): print(output) break else: - print('success: Slave VRRP address was added
') + print('success: Slave VRRP address has been added
') os.system("rm -f %s" % script) @@ -1127,7 +1126,7 @@ if form.getvalue('install_grafana'): l = l.split('"')[1] print(l + "
") break - except: + except Exception: print(output) break else: @@ -1182,7 +1181,7 @@ if form.getvalue('haproxy_exp_install'): l = l.split('"')[1] print(l + "
") break - except: + except Exception: print(output) break else: @@ -1232,7 +1231,7 @@ if form.getvalue('nginx_exp_install'): l = l.split('"')[1] print(l + "
") break - except: + except Exception: print(output) break else: @@ -1288,7 +1287,7 @@ if form.getvalue('backup') or form.getvalue('deljob') or form.getvalue('backupup l = l.split('"')[1] print('error: ' + l + "
") break - except: + except Exception: print('error: ' + output) break else: @@ -1674,12 +1673,13 @@ if form.getvalue('newserver') is not None: funct.logging('a new server ' + hostname, ' has created ', haproxywi=1, login=1) if form.getvalue('updatehapwiserver') is not None: - id = form.getvalue('updatehapwiserver') + hapwi_id = form.getvalue('updatehapwiserver') active = form.getvalue('active') name = form.getvalue('name') alert = form.getvalue('alert_en') metrics = form.getvalue('metrics') - sql.update_hapwi_server(id, alert, metrics, active) + service_name = form.getvalue('service_name') + sql.update_hapwi_server(hapwi_id, alert, metrics, active, service_name) funct.logging('the server ' + name, ' has updated ', haproxywi=1, login=1) if form.getvalue('updateserver') is not None: @@ -1690,7 +1690,7 @@ if form.getvalue('updateserver') is not None: nginx = form.getvalue('nginx') enable = form.getvalue('enable') master = form.getvalue('slave') - id = form.getvalue('id') + serv_id = form.getvalue('id') cred = form.getvalue('cred') port = form.getvalue('port') desc = form.getvalue('desc') @@ -1698,7 +1698,7 @@ if form.getvalue('updateserver') is not None: if name is None or port is None: print(error_mess) else: - sql.update_server(name, group, typeip, enable, master, id, cred, port, desc, haproxy, nginx) + sql.update_server(name, group, typeip, enable, master, serv_id, cred, port, desc, haproxy, nginx) funct.logging('the server ' + name, ' has updated ', haproxywi=1, login=1) if form.getvalue('serverdel') is not None: @@ -1743,14 +1743,14 @@ if form.getvalue('groupdel') is not None: if form.getvalue('updategroup') is not None: name = form.getvalue('updategroup') descript = form.getvalue('descript') - id = form.getvalue('id') + group_id = form.getvalue('id') if name is None: print(error_mess) else: - group = sql.select_groups(id=id) + group = sql.select_groups(id=group_id) for g in group: groupname = g[1] - sql.update_group(name, descript, id) + sql.update_group(name, descript, group_id) funct.logging('the group ' + groupname, ' has update ', haproxywi=1, login=1) if form.getvalue('new_ssh'): @@ -1797,7 +1797,7 @@ if form.getvalue('sshdel') is not None: funct.logging(name, ' has deleted the SSH credentials ', haproxywi=1, login=1) if form.getvalue('updatessh'): - id = form.getvalue('id') + ssh_id = form.getvalue('id') name = form.getvalue('name') enable = form.getvalue('ssh_enable') group = form.getvalue('group') @@ -1810,7 +1810,7 @@ if form.getvalue('updatessh'): fullpath = funct.get_config_var('main', 'fullpath') - for sshs in sql.select_ssh(id=id): + for sshs in sql.select_ssh(id=ssh_id): ssh_enable = sshs[2] ssh_key_name = fullpath + '/keys/%s.pem' % sshs[1] new_ssh_key_name = fullpath + '/keys/%s.pem' % name @@ -1821,9 +1821,9 @@ if form.getvalue('updatessh'): try: funct.subprocess_execute(cmd) funct.subprocess_execute(cmd1) - except: + except Exception: pass - sql.update_ssh(id, name, enable, group, username, password) + sql.update_ssh(ssh_id, name, enable, group, username, password) funct.logging('the SSH ' + name, ' has updated credentials ', haproxywi=1, login=1) if form.getvalue('ssh_cert'): @@ -1900,11 +1900,11 @@ if form.getvalue('updatetoken') is not None: token = form.getvalue('updatetoken') channel = form.getvalue('updategchanel') group = form.getvalue('updatetelegramgroup') - id = form.getvalue('id') + user_id = form.getvalue('id') if token is None or channel is None or group is None: print(error_mess) else: - sql.update_telegram(token, channel, group, id) + sql.update_telegram(token, channel, group, user_id) funct.logging('group ' + group, ' telegram token has updated channel: ' + channel, haproxywi=1, login=1) if form.getvalue('updatesettings') is not None: diff --git a/app/overview.py b/app/overview.py index 90d491fe..93405493 100644 --- a/app/overview.py +++ b/app/overview.py @@ -101,35 +101,35 @@ except Exception as e: is_metrics_worker = '' token = '' -template = template.render(h2 = 1, - autorefresh = 1, - title = "Overview", - role = role, - user = user, - users = users, - groups = groups, - users_groups = sql.select_user_groups_with_names(1, all=1), - roles = sql.select_roles(), - metrics_master = ''.join(metrics_master), - metrics_worker = ''.join(metrics_worker), - checker_master = ''.join(checker_master), - checker_worker = ''.join(checker_worker), - keep_alive = ''.join(keep_alive), - smon = ''.join(smon), - grafana = ''.join(grafana), - prometheus = ''.join(prometheus), - haproxy_wi_log_id = funct.haproxy_wi_log(log_id=1, file="haproxy-wi-", with_date=1), - metrics_log_id = funct.haproxy_wi_log(log_id=1, file="metrics-", with_date=1), - checker_log_id = funct.haproxy_wi_log(log_id=1, file="checker-", with_date=1), - keep_alive_log_id = funct.haproxy_wi_log(log_id=1, file="keep_alive"), - checker_error_log_id = funct.haproxy_wi_log(log_id=1, file="checker-error"), - metrics_error_log_id = funct.haproxy_wi_log(log_id=1, file="metrics-error"), - error = stderr, - versions = funct.versions(), - haproxy_wi_log = funct.haproxy_wi_log(), - servers = servers, - is_checker_worker = is_checker_worker, - is_metrics_worker = is_metrics_worker, - host = host, - token = token) -print(template) \ No newline at end of file +template = template.render(h2=1, + autorefresh=1, + title="Overview", + role=role, + user=user, + users=users, + groups=groups, + users_groups=sql.select_user_groups_with_names(1, all=1), + roles=sql.select_roles(), + metrics_master=''.join(metrics_master), + metrics_worker=''.join(metrics_worker), + checker_master=''.join(checker_master), + checker_worker=''.join(checker_worker), + keep_alive=''.join(keep_alive), + smon=''.join(smon), + grafana=''.join(grafana), + prometheus=''.join(prometheus), + haproxy_wi_log_id=funct.haproxy_wi_log(log_id=1, file="haproxy-wi-", with_date=1), + metrics_log_id=funct.haproxy_wi_log(log_id=1, file="metrics-", with_date=1), + checker_log_id=funct.haproxy_wi_log(log_id=1, file="checker-", with_date=1), + keep_alive_log_id=funct.haproxy_wi_log(log_id=1, file="keep_alive"), + checker_error_log_id=funct.haproxy_wi_log(log_id=1, file="checker-error"), + metrics_error_log_id=funct.haproxy_wi_log(log_id=1, file="metrics-error"), + error=stderr, + versions=funct.versions(), + haproxy_wi_log=funct.haproxy_wi_log(), + servers=servers, + is_checker_worker=is_checker_worker, + is_metrics_worker=is_metrics_worker, + host=host, + token=token) +print(template) diff --git a/app/sql.py b/app/sql.py index b70102d2..dc7fe8bf 100644 --- a/app/sql.py +++ b/app/sql.py @@ -201,7 +201,6 @@ def add_setting_for_new_group(group_id): sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('server_state_file', '/etc/haproxy/haproxy.state', 'haproxy', 'Path to HAProxy state file','" + group_id + "');") sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('haproxy_sock', '/var/run/haproxy.sock', 'haproxy', 'Path to HAProxy sock file','" + group_id + "');") sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('haproxy_sock_port', '1999', 'haproxy', 'HAProxy sock port','" + group_id + "');") - sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('firewall_enable', '0', 'haproxy', 'If enable this option Haproxy-wi will be configure firewalld based on config port','" + group_id + "');") sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('nginx_path_error_logs', '/var/log/nginx/error.log', 'nginx', 'Nginx error log','" + group_id + "');") sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('nginx_stats_user', 'admin', 'nginx', 'Username for Stats web page Nginx','" + group_id + "');") sql.append("INSERT INTO settings (param, value, section, `desc`, `group`) values('nginx_stats_password', 'password', 'nginx', 'Password for Stats web page Nginx','" + group_id + "');") @@ -326,13 +325,17 @@ def delete_server(id): con.close() -def update_hapwi_server(id, alert, metrics, active): +def update_hapwi_server(server_id, alert, metrics, active, service_name): con, cur = get_cur() + updated_service = 'active' + if service_name == 'nginx': + updated_service = 'nginx_active' + sql = """ update servers set alert = '%s', metrics = '%s', - active = '%s' - where id = '%s'""" % (alert, metrics, active, id) + '%s' = '%s' + where id = '%s'""" % (alert, metrics, updated_service, active, server_id) try: cur.execute(sql) con.commit() @@ -1269,10 +1272,32 @@ def select_waf_servers_metrics(uuid, **kwargs): def select_waf_metrics(serv, **kwargs): con, cur = get_cur() + if mysql_enable == '1': - sql = """ select * from waf_metrics where serv = '%s' order by `date` desc limit 60 """ % serv + if kwargs.get('time_range') == '60': + date_from = "and date > now() - INTERVAL 60 minute and rowid % 2 = 0" + elif kwargs.get('time_range') == '180': + date_from = "and date > now() - INTERVAL 180 minute and rowid % 5 = 0" + elif kwargs.get('time_range') == '360': + date_from = "and date > now() - INTERVAL 360 minute and rowid % 7 = 0" + elif kwargs.get('time_range') == '720': + date_from = "and date > now() - INTERVAL 720 minute and rowid % 9 = 0" + else: + date_from = "and date > now() - INTERVAL 30 minute" + sql = """ select * from waf_metrics where serv = '{serv}' {date_from} order by `date` desc limit 60 """.format(serv=serv, date_from=date_from) else: - sql = """ select * from (select * from waf_metrics where serv = '%s' order by `date` desc limit 60) order by `date`""" % serv + if kwargs.get('time_range') == '60': + date_from = "and date > datetime('now', '-60 minutes', 'localtime') and rowid % 2 = 0" + elif kwargs.get('time_range') == '180': + date_from = "and date > datetime('now', '-180 minutes', 'localtime') and rowid % 5 = 0" + elif kwargs.get('time_range') == '360': + date_from = "and date > datetime('now', '-360 minutes', 'localtime') and rowid % 7 = 0" + elif kwargs.get('time_range') == '720': + date_from = "and date > datetime('now', '-720 minutes', 'localtime') and rowid % 9 = 0" + else: + date_from = "and date > datetime('now', '-30 minutes', 'localtime')" + sql = """ select * from (select * from waf_metrics where serv = '{serv}' {date_from} order by `date`) order by `date` """.format(serv=serv, date_from=date_from) + try: cur.execute(sql) except sqltool.Error as e: @@ -1451,26 +1476,26 @@ def select_metrics(serv, **kwargs): con, cur = get_cur() if mysql_enable == '1': - if kwargs.get('time_range') == 60: + if kwargs.get('time_range') == '60': date_from = "and date > now() - INTERVAL 60 minute and rowid % 2 = 0" - elif kwargs.get('time_range') == 180: + elif kwargs.get('time_range') == '180': date_from = "and date > now() - INTERVAL 180 minute and rowid % 5 = 0" - elif kwargs.get('time_range') == 720: + elif kwargs.get('time_range') == '360': + date_from = "and date > now() - INTERVAL 360 minute and rowid % 7 = 0" + elif kwargs.get('time_range') == '720': date_from = "and date > now() - INTERVAL 720 minute and rowid % 9 = 0" - elif kwargs.get('time_range') == 1440: - date_from = "and date > now() - INTERVAL 1440 minute and rowid % 17 = 0" else: date_from = "and date > now() - INTERVAL 30 minute" sql = """ select * from metrics where serv = '{serv}' {date_from} order by `date` desc limit 60 """.format(serv=serv, date_from=date_from) else: - if kwargs.get('time_range') == 60: + if kwargs.get('time_range') == '60': date_from = "and date > datetime('now', '-60 minutes', 'localtime') and rowid % 2 = 0" - elif kwargs.get('time_range') == 180: + elif kwargs.get('time_range') == '180': date_from = "and date > datetime('now', '-180 minutes', 'localtime') and rowid % 5 = 0" - elif kwargs.get('time_range') == 720: + elif kwargs.get('time_range') == '360': + date_from = "and date > datetime('now', '-360 minutes', 'localtime') and rowid % 7 = 0" + elif kwargs.get('time_range') == '720': date_from = "and date > datetime('now', '-720 minutes', 'localtime') and rowid % 9 = 0" - elif kwargs.get('time_range') == 1440: - date_from = "and date > datetime('now', '-1440 minutes', 'localtime') and rowid % 17 = 0" else: date_from = "and date > datetime('now', '-30 minutes', 'localtime')" @@ -2052,13 +2077,25 @@ def update_smon(id, ip, port, body, telegram, group, desc, en): con.close() +def alerts_history(service, user_group): + con, cur = get_cur() + sql = """ select message, level, ip, port, date from alerts + where service = '%s' and user_group = '%s' order by date desc; """ % (service, user_group) + try: + cur.execute(sql) + except sqltool.Error as e: + funct.out_error(e) + else: + return cur.fetchall() + + def select_en_service(): con, cur = get_cur() sql = """ select ip, port, telegram_channel_id, id, user_group from smon where en = 1""" try: cur.execute(sql) except sqltool.Error as e: - out_error(e) + funct.out_error(e) else: return cur.fetchall() @@ -2069,7 +2106,7 @@ def select_status(id): try: cur.execute(sql) except sqltool.Error as e: - print("An error occurred:", e) + funct.out_error(e) else: for status in cur: return status[0] diff --git a/app/templates/base.html b/app/templates/base.html index 3c94cd21..bdeeb9b3 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -99,6 +99,7 @@ SMON