From 8e9b5f861a188822e5560f42c1f67fa07dc63523 Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Sun, 3 Jul 2022 09:11:34 +0300 Subject: [PATCH] v6.1.1.0 Change log: https://roxy-wi.org/changelog.py#6_1_1 --- app/create_db.py | 6 +- app/db_model.py | 14 +++- app/funct.py | 9 +-- app/metrics.py | 5 ++ app/options.py | 39 ++++------- app/scripts/ansible/roles/backup.yml | 12 ++-- app/sql.py | 92 ++++++++++++++++++++++++-- app/templates/ajax/show_users_ovw.html | 5 ++ app/templates/base.html | 2 + app/templates/hapservers.html | 29 +++++--- app/templates/metrics.html | 36 ++++++---- app/templates/ovw.html | 13 ++-- inc/metrics.js | 76 +++++++++++++++++++++ inc/script.js | 4 ++ 14 files changed, 265 insertions(+), 77 deletions(-) diff --git a/app/create_db.py b/app/create_db.py index c481ee56..7679fe3e 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -33,7 +33,7 @@ def default_values(): {'param': 'haproxy_path_logs', 'value': '/var/log/haproxy/', 'section': 'haproxy', 'desc': 'The path for HAProxy logs', 'group': '1'}, {'param': 'syslog_server_enable', 'value': '0', 'section': 'logs', - 'desc': 'Enable getting logs from a syslog server; (0 - no, 1 - yes)', 'group': '1'}, + 'desc': 'Enable getting logs from a syslog server', 'group': '1'}, {'param': 'syslog_server', 'value': '', 'section': 'logs', 'desc': 'IP address of the syslog_server', 'group': '1'}, {'param': 'log_time_storage', 'value': '14', 'section': 'logs', @@ -59,7 +59,7 @@ def default_values(): {'param': 'haproxy_sock_port', 'value': '1999', 'section': 'haproxy', 'desc': 'HAProxy sock port', 'group': '1'}, {'param': 'apache_log_path', 'value': '/var/log/' + apache_dir + '/', 'section': 'logs', - 'desc': 'Path to Apache logs. Apache service for Roxy-WI', 'group': '1'}, + 'desc': 'Path to Apache logs. Apache service for Roxy-WI', 'group': '1'}, {'param': 'nginx_path_logs', 'value': '/var/log/nginx/', 'section': 'nginx', 'desc': 'The path for NGINX logs', 'group': '1'}, {'param': 'nginx_stats_user', 'value': 'admin', 'section': 'nginx', 'desc': 'Username for accessing NGINX stats page', @@ -744,7 +744,7 @@ def update_db_v_6_1_0(**kwargs): def update_ver(): - query = Version.update(version='6.1.0.0') + query = Version.update(version='6.1.1.0') try: query.execute() except Exception: diff --git a/app/db_model.py b/app/db_model.py index 66fba7ed..42856f91 100644 --- a/app/db_model.py +++ b/app/db_model.py @@ -226,6 +226,16 @@ class NginxMetrics(BaseModel): primary_key = False +class ApacheMetrics(BaseModel): + serv = CharField() + conn = IntegerField() + date = DateTimeField(default=datetime.now) + + class Meta: + table_name = 'apache_metrics' + primary_key = False + + class Version(BaseModel): version = CharField() @@ -268,7 +278,7 @@ class WafRules(BaseModel): serv = CharField() rule_name = CharField() rule_file = CharField() - desc = CharField(null=True) + desc = TextField(null=True) en = IntegerField(constraints=[SQL('DEFAULT 1')]) class Meta: @@ -532,4 +542,4 @@ def create_tables(): Setting, Cred, Backup, Metrics, WafMetrics, Version, Option, SavedServer, Waf, ActionHistory, PortScannerSettings, PortScannerPorts, PortScannerHistory, ProvidersCreds, ServiceSetting, ProvisionedServers, MetricsHttpStatus, SMON, WafRules, Alerts, GeoipCodes, NginxMetrics, - SystemInfo, Services, UserName, GitSetting, CheckerSetting]) + SystemInfo, Services, UserName, GitSetting, CheckerSetting, ApacheMetrics]) diff --git a/app/funct.py b/app/funct.py index 2f96850d..652f2f94 100644 --- a/app/funct.py +++ b/app/funct.py @@ -712,7 +712,7 @@ def install_haproxy(server_ip, **kwargs): os.system("cp scripts/%s ." % script) if haproxy_ver is None: - haproxy_ver = '2.5.1-1' + haproxy_ver = '2.6.0-1' if proxy is not None and proxy != '' and proxy != 'None': proxy_serv = proxy @@ -1330,16 +1330,13 @@ def show_haproxy_log(serv, rows=10, waf='0', grep=None, hour='00', minut='00', h return show_log(output, grep=grep) -def haproxy_wi_log(**kwargs): +def roxy_wi_log(**kwargs): log_path = get_config_var('main', 'log_path') if kwargs.get('log_id'): selects = get_files(log_path, format="log") for key, value in selects: - if kwargs.get('with_date'): - log_file = kwargs.get('file') + get_data('logs') + ".log" - else: - log_file = kwargs.get('file') + ".log" + log_file = kwargs.get('file') + ".log" if log_file == value: return key else: diff --git a/app/metrics.py b/app/metrics.py index df7129e6..7669d79b 100644 --- a/app/metrics.py +++ b/app/metrics.py @@ -29,10 +29,15 @@ try: if funct.check_login(service=2): title = "NGINX`s metrics" servers = sql.select_nginx_servers_metrics_for_master() + elif service == 'apache': + if funct.check_login(service=4): + title = "Apache`s metrics" + servers = sql.select_apache_servers_metrics_for_master() else: if funct.check_login(service=1): title = "HAProxy`s metrics" servers = sql.select_servers_metrics() + service = 'haproxy' services = '1' except Exception: pass diff --git a/app/options.py b/app/options.py index d2def109..2330b8f2 100644 --- a/app/options.py +++ b/app/options.py @@ -15,6 +15,7 @@ if ( or form.getvalue('new_http_metrics') or form.getvalue('new_waf_metrics') or form.getvalue('new_nginx_metrics') + or form.getvalue('new_apache_metrics') or form.getvalue('metrics_hapwi_ram') or form.getvalue('metrics_hapwi_cpu') or form.getvalue('getoption') @@ -1335,7 +1336,10 @@ if form.getvalue('master'): group_id = sql.get_group_id_by_server_ip(master) cred_id = sql.get_cred_id_by_server_ip(master) hostname = sql.get_hostname_by_server_ip(master) - sql.add_server(hostname + '-VIP', IP, group_id, '1', '1', '0', cred_id, ssh_port, 'VRRP IP for ' + master, haproxy, nginx, '0') + firewall = 1 if funct.is_service_active(master, 'firewalld') else 0 + sql.add_server( + hostname + '-VIP', IP, group_id, '1', '1', '0', cred_id, ssh_port, 'VRRP IP for ' + master, haproxy, nginx, '0', firewall + ) if form.getvalue('master_slave'): master = form.getvalue('master_slave') @@ -1934,35 +1938,16 @@ if form.getvalue('new_http_metrics'): print(json.dumps(metrics)) -if form.getvalue('new_waf_metrics'): +if any((form.getvalue('new_nginx_metrics'), form.getvalue('new_apache_metrics'), form.getvalue('new_waf_metrics'))): serv = form.getvalue('server') hostname = sql.get_hostname_by_server_ip(serv) time_range = form.getvalue('time_range') - metric = sql.select_waf_metrics(serv, time_range=time_range) - metrics = {'chartData': {}} - metrics['chartData']['labels'] = {} - labels = '' - curr_con = '' - - for i in metric: - label = str(i[2]) - label = label.split(' ')[1] - labels += label + ',' - curr_con += str(i[1]) + ',' - - metrics['chartData']['labels'] = labels - metrics['chartData']['curr_con'] = curr_con - metrics['chartData']['server'] = hostname + ' (' + serv + ')' - - import json - - print(json.dumps(metrics)) - -if form.getvalue('new_nginx_metrics'): - serv = form.getvalue('server') - hostname = sql.get_hostname_by_server_ip(serv) - time_range = form.getvalue('time_range') - metric = sql.select_nginx_metrics(serv, time_range=time_range) + if form.getvalue('new_nginx_metrics'): + metric = sql.select_nginx_metrics(serv, time_range=time_range) + elif form.getvalue('new_apache_metrics'): + metric = sql.select_apache_metrics(serv, time_range=time_range) + elif form.getvalue('new_waf_metrics'): + metric = sql.select_waf_metrics(serv, time_range=time_range) metrics = {'chartData': {}} metrics['chartData']['labels'] = {} labels = '' diff --git a/app/scripts/ansible/roles/backup.yml b/app/scripts/ansible/roles/backup.yml index 82dc7b01..ee9aa8fb 100644 --- a/app/scripts/ansible/roles/backup.yml +++ b/app/scripts/ansible/roles/backup.yml @@ -9,7 +9,7 @@ - name: Creates directory file: - path: "{{ RPATH }}/haproxy-wi-configs-backup/configs" + path: "{{ RPATH }}/roxy-wi-configs-backup/configs" state: directory owner: "{{ ansible_user }}" when: DELJOB|length == 0 @@ -22,21 +22,23 @@ - name: Creates backup jobs cron: - name: "Backup HAProxy configs for user {{ SERVER }} {{ item }}" + name: "Roxy-WI Backup configs for server {{ SERVER }} {{ item }}" special_time: "{{ TIME }}" - job: "rsync -arv {{ TYPE }} /var/www/haproxy-wi/configs/{{ item }}/{{ SERVER }}* {{ ansible_user }}@{{ HOST }}:{{ RPATH }}/haproxy-wi-configs-backup/configs/{{ item }} -e 'ssh -i {{ KEY }} -o StrictHostKeyChecking=no' --log-file=/var/www/haproxy-wi/log/backup.log" + job: "rsync -arv {{ TYPE }} /var/www/haproxy-wi/configs/{{ item }}/{{ SERVER }}* {{ ansible_user }}@{{ HOST }}:{{ RPATH }}/roxy-wi-configs-backup/configs/{{ item }} -e 'ssh -i {{ KEY }} -o StrictHostKeyChecking=no' --log-file=/var/www/haproxy-wi/log/backup.log" when: DELJOB|length == 0 with_items: - kp_config - hap_config - nginx_config + - apache_config - name: Removes backup jobs cron: - name: "Backup HAProxy configs for user {{ SERVER }} {{ item }}" + name: "Roxy-WI Backup configs for server {{ SERVER }} {{ item }}" state: absent when: DELJOB|length > 0 with_items: - kp_config - hap_config - - nginx_config \ No newline at end of file + - nginx_config + - apache_config diff --git a/app/sql.py b/app/sql.py index c21309da..31203120 100755 --- a/app/sql.py +++ b/app/sql.py @@ -297,7 +297,7 @@ def update_hapwi_server(server_id, alert, metrics, active, service_name): update_hapwi = Server.update(keepalived_alert=alert, keepalived_active=active).where( Server.server_id == server_id) elif service_name == 'apache': - update_hapwi = Server.update(apache_alert=alert, apache_active=active).where( + update_hapwi = Server.update(apache_alert=alert, apache_active=active, apache_metrics=metrics).where( Server.server_id == server_id) else: update_hapwi = Server.update(alert=alert, metrics=metrics, active=active).where( @@ -872,7 +872,8 @@ def get_dick_permit(**kwargs): try: cursor.execute(sql) except Exception as e: - out_error(e) + # out_error(e) + pass else: return cursor.fetchall() @@ -1233,6 +1234,13 @@ def insert_nginx_metrics(serv, conn): out_error(e) +def insert_apache_metrics(serv, conn): + try: + ApacheMetrics.insert(serv=serv, conn=conn, date=funct.get_data('regular')).execute() + except Exception as e: + out_error(e) + + def select_waf_metrics_enable_server(ip): query = Waf.select(Waf.metrics).join(Server, on=(Waf.server_id == Server.server_id)).where(Server.ip == ip) try: @@ -1365,6 +1373,44 @@ def select_nginx_metrics(serv, **kwargs): return cursor.fetchall() +def select_apache_metrics(serv, **kwargs): + cursor = conn.cursor() + + if mysql_enable == '1': + if kwargs.get('time_range') == '60': + date_from = "and date > now() - INTERVAL 60 minute group by `date` div 100" + elif kwargs.get('time_range') == '180': + date_from = "and date > now() - INTERVAL 180 minute group by `date` div 200" + elif kwargs.get('time_range') == '360': + date_from = "and date > now() - INTERVAL 360 minute group by `date` div 300" + elif kwargs.get('time_range') == '720': + date_from = "and date > now() - INTERVAL 720 minute group by `date` div 500" + else: + date_from = "and date > now() - INTERVAL 30 minute" + sql = """ select * from apache_metrics where serv = '{serv}' {date_from} order by `date` desc limit 60 """.format( + serv=serv, date_from=date_from) + else: + if kwargs.get('time_range') == '60': + date_from = "and date > datetime('now', '-60 minutes', 'localtime') and rowid % 2 = 0" + elif kwargs.get('time_range') == '180': + date_from = "and date > datetime('now', '-180 minutes', 'localtime') and rowid % 5 = 0" + elif kwargs.get('time_range') == '360': + date_from = "and date > datetime('now', '-360 minutes', 'localtime') and rowid % 7 = 0" + elif kwargs.get('time_range') == '720': + date_from = "and date > datetime('now', '-720 minutes', 'localtime') and rowid % 9 = 0" + else: + date_from = "and date > datetime('now', '-30 minutes', 'localtime')" + sql = """ select * from (select * from apache_metrics where serv = '{serv}' {date_from} order by `date`) order by `date` """.format( + serv=serv, date_from=date_from) + + try: + cursor.execute(sql) + except Exception as e: + out_error(e) + else: + return cursor.fetchall() + + def insert_waf_metrics_enable(serv, enable): try: server_id = Server.get(Server.ip == serv).server_id @@ -1544,6 +1590,14 @@ def delete_nginx_metrics(): out_error(e) +def delete_apache_metrics(): + query = ApacheMetrics.delete().where(ApacheMetrics.date < funct.get_data('regular', timedelta_minus=3)) + try: + query.execute() + except Exception as e: + out_error(e) + + def select_metrics(serv, **kwargs): cursor = conn.cursor() @@ -1624,10 +1678,28 @@ def select_metrics_http(serv, **kwargs): def select_servers_metrics_for_master(**kwargs): - if kwargs.get('group') is not None: - query = Server.select(Server.ip).where((Server.metrics == 1) & (Server.groups == kwargs.get('group'))) + if kwargs.get('group') != 1: + query = Server.select(Server.ip).where( + ((Server.metrics == 1) | (Server.nginx_metrics == 1) | (Server.apache_metrics == 1)) + & (Server.groups == kwargs.get('group')) + ) else: - query = Server.select(Server.ip).where(Server.metrics == 1) + query = Server.select(Server.ip).where( + (Server.metrics == 1) + | (Server.nginx_metrics == 1) + | (Server.apache_metrics == 1) + ) + + try: + query_res = query.execute() + except Exception as e: + out_error(e) + else: + return query_res + + +def select_haproxy_servers_metrics_for_master(): + query = Server.select(Server.ip).where(Server.metrics == 1) try: query_res = query.execute() except Exception as e: @@ -1646,6 +1718,16 @@ def select_nginx_servers_metrics_for_master(): return query_res +def select_apache_servers_metrics_for_master(): + query = Server.select(Server.ip).where(Server.apache_metrics == 1) + try: + query_res = query.execute() + except Exception as e: + out_error(e) + else: + return query_res + + def select_servers_metrics(): group_id = funct.get_user_group(id=1) if funct.check_user_group(): diff --git a/app/templates/ajax/show_users_ovw.html b/app/templates/ajax/show_users_ovw.html index 58238b51..b3f15402 100644 --- a/app/templates/ajax/show_users_ovw.html +++ b/app/templates/ajax/show_users_ovw.html @@ -55,3 +55,8 @@ {% endif %} {% endfor %} +{% if users|length > 3 %} + +{% endif %} diff --git a/app/templates/base.html b/app/templates/base.html index 823f05a3..96c23f74 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -103,7 +103,9 @@