From f696d8ed6320fb050d38eb0f90b6b41e7144897f Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Wed, 2 Mar 2022 23:09:39 +0300 Subject: [PATCH] v5.4.3.0 Changelog: https://roxy-wi.org/changelog.py#5_4_3 --- app/config.py | 11 +- app/create_db.py | 36 ++- app/funct.py | 112 ++++++- app/logs.py | 3 + app/options.py | 86 +++-- .../ansible/roles/haproxy/tasks/main.yml | 4 +- app/scripts/ansible/roles/waf/tasks/main.yml | 12 + .../ajax/haproxyservers_backends.html | 9 + app/templates/ajax/show_configs_files.html | 83 ++++- app/templates/ajax/show_log_files.html | 20 ++ app/templates/ajax/show_system_info.html | 2 +- .../ajax/show_user_current_group.html | 26 +- app/templates/base.html | 2 + app/templates/config.html | 9 +- app/templates/logs.html | 45 ++- app/templates/waf.html | 1 + inc/codemirror/dialog.css | 32 ++ inc/codemirror/dialog.js | 163 ++++++++++ inc/codemirror/docs.css | 274 ++++++++++++++++ inc/codemirror/haproxy.js | 2 +- inc/codemirror/jump-to-line.js | 53 +++ inc/codemirror/modsec.js | 155 +++++++++ inc/codemirror/search.js | 292 +++++++++++++++++ inc/codemirror/searchcursor.js | 305 ++++++++++++++++++ inc/script.js | 92 +++++- inc/select2.css | 1 + inc/select2.js | 1 + inc/users.js | 2 +- index.html | 2 + 29 files changed, 1732 insertions(+), 103 deletions(-) create mode 100644 app/templates/ajax/show_log_files.html create mode 100644 inc/codemirror/dialog.css create mode 100644 inc/codemirror/dialog.js create mode 100644 inc/codemirror/docs.css create mode 100644 inc/codemirror/jump-to-line.js create mode 100644 inc/codemirror/modsec.js create mode 100644 inc/codemirror/search.js create mode 100644 inc/codemirror/searchcursor.js create mode 100644 inc/select2.css create mode 100644 inc/select2.js diff --git a/app/config.py b/app/config.py index 3fd0c1f1..4b75f650 100644 --- a/app/config.py +++ b/app/config.py @@ -50,9 +50,13 @@ else: servers = sql.get_dick_permit() if serv is not None: - cfg = configs_dir + serv + "-" + funct.get_data('config') + "."+file_format + if service == 'nginx': + conf_file_name_short = config_file_name.split('/')[-1] + cfg = configs_dir + serv + "-" + conf_file_name_short + "-" + funct.get_data('config') + "." + file_format + else: + cfg = configs_dir + serv + "-" + funct.get_data('config') + "."+file_format -if serv is not None and form.getvalue('open') is not None: +if serv is not None and form.getvalue('open') is not None and form.getvalue('new_config') is None: funct.check_is_server_in_group(serv) if service == 'keepalived': error = funct.get_config(serv, cfg, keepalived=1) @@ -82,6 +86,9 @@ if serv is not None and form.getvalue('open') is not None: os.system("/bin/mv %s %s.old" % (cfg, cfg)) +if form.getvalue('new_config') is not None: + config_read = ' ' + if serv is not None and form.getvalue('config') is not None: import sys funct.check_is_server_in_group(serv) diff --git a/app/create_db.py b/app/create_db.py index 6f58d153..a5bab14c 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -29,8 +29,8 @@ def default_values(): {'param': 'lists_path', 'value': 'lists', 'section': 'main', 'desc': 'Path to the black and the wild list. The value of this paramer should be specified as a relative path beginning with $HOME_ROXY-WI', 'group': '1'}, - {'param': 'haproxy_path_logs', 'value': '/var/log/haproxy/access.log', 'section': 'haproxy', - 'desc': 'The default local path for saving logs', 'group': '1'}, + {'param': 'haproxy_path_logs', 'value': '/var/log/haproxy/', 'section': 'haproxy', + 'desc': 'The path for HAProxy logs', 'group': '1'}, {'param': 'syslog_server_enable', 'value': '0', 'section': 'logs', 'desc': 'Enable getting logs from a syslog server; (0 - no, 1 - yes)', 'group': '1'}, {'param': 'syslog_server', 'value': '', 'section': 'logs', 'desc': 'IP address of the syslog_server', @@ -57,8 +57,8 @@ def default_values(): 'group': '1'}, {'param': 'apache_log_path', 'value': '/var/log/'+apache_dir+'/', 'section': 'logs', 'desc': 'Path to Apache logs', 'group': '1'}, - {'param': 'nginx_path_error_logs', 'value': '/var/log/nginx/error.log', 'section': 'nginx', - 'desc': 'Nginx error log', 'group': '1'}, + {'param': 'nginx_path_logs', 'value': '/var/log/nginx/', 'section': 'nginx', + 'desc': 'The path for Nginx logs', 'group': '1'}, {'param': 'nginx_stats_user', 'value': 'admin', 'section': 'nginx', 'desc': 'Username for accessing Nginx stats page', 'group': '1'}, {'param': 'nginx_stats_password', 'value': 'password', 'section': 'nginx', @@ -873,8 +873,30 @@ def update_db_v_5_4_2(**kwargs): print("Updating... DB has been updated to version 5.4.2") +def update_db_v_5_4_3(**kwargs): + query = Setting.update(param='nginx_path_logs', value='/var/log/nginx/').where(Setting.param == 'nginx_path_error_logs') + try: + query.execute() + except Exception as e: + print("An error occurred:", e) + else: + if kwargs.get('silent') != 1: + print("Updating... DB has been updated to version 5.4.3") + + +def update_db_v_5_4_3_1(**kwargs): + query = Setting.update( value='/etc/nginx/').where(Setting.param == 'nginx_dir') + try: + query.execute() + except Exception as e: + print("An error occurred:", e) + else: + if kwargs.get('silent') != 1: + print("Updating... DB has been updated to version 5.4.3-1") + + def update_ver(): - query = Version.update(version='5.4.2.0') + query = Version.update(version='5.4.3.0') try: query.execute() except: @@ -911,6 +933,8 @@ def update_all(): update_db_v_5_3_2() update_db_v_5_3_2_2() update_db_v_5_4_2() + update_db_v_5_4_3() + update_db_v_5_4_3_1() update_ver() @@ -944,6 +968,8 @@ def update_all_silent(): update_db_v_5_3_2(silent=1) update_db_v_5_3_2_2(silent=1) update_db_v_5_4_2(silent=1) + update_db_v_5_4_3(silent=1) + update_db_v_5_4_3_1(silent=1) update_ver() diff --git a/app/funct.py b/app/funct.py index 765edcd5..eadc36f1 100644 --- a/app/funct.py +++ b/app/funct.py @@ -490,19 +490,29 @@ def diff_config(oldcfg, cfg, **kwargs): except IOError: print('
Can\'t read write change to log. %s
' % stderr) pass + + +def get_remote_sections(server_ip: str, service: str) -> str: + import sql + remote_dir = service+'_dir' + config_dir = sql.get_setting(remote_dir) + config_dir = return_nice_path(config_dir) + if service == 'nginx': + section_name = 'server_name' + elif service == 'apache': + section_name = 'ServerName' + commands = ['sudo grep {} {}* -R |grep -v \'$server_name\|#\'|awk \'{{print $1, $3}}\''.format(section_name, config_dir)] + + backends = ssh_command(server_ip, commands) + + return backends def get_sections(config, **kwargs): return_config = list() with open(config, 'r') as f: for line in f: - if kwargs.get('service') == 'nginx': - if 'server_name' in line: - line = line.split('server_name')[1] - line = line.split(';')[0] - line = line.strip() - return_config.append(line) - elif kwargs.get('service') == 'keepalived': + if kwargs.get('service') == 'keepalived': import re ip_pattern = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}') find_ip = re.findall(ip_pattern,line) @@ -658,7 +668,7 @@ def get_stick_table(table): def show_installation_output(error, output, service): - if error and "WARNING" not in error: + if error and "DEPRECATION WARNING" not in error: logging('localhost', error, haproxywi=1) print('error: '+error) return False @@ -1168,9 +1178,35 @@ def show_log(stdout, **kwargs): return out +def show_finding_in_config(stdout: str, **kwargs) -> str: + i = 0 + out = '' + grep = '' + line_class = 'line' + + if kwargs.get('grep'): + import re + grep = kwargs.get('grep') + grep = re.sub(r'[?|$|!|^|*|\]|\[|,| |]', r'', grep) + + out += '
--
' + for line in stdout: + i = i + 1 + if kwargs.get('grep'): + line = line.replace(grep, ''+grep+'') + line_class = "line" if '--' in line else "line3" + out += '
' + line + '
' + + out += '
--
' + + return out + + + def show_haproxy_log(serv, rows=10, waf='0', grep=None, hour='00', minut='00', hour1='24', minut1='00', service='haproxy', **kwargs): import sql exgrep = form.getvalue('exgrep') + log_file = form.getvalue('file') date = hour+':'+minut date1 = hour1+':'+minut1 cmd = '' @@ -1189,11 +1225,11 @@ def show_haproxy_log(serv, rows=10, waf='0', grep=None, hour='00', minut='00', h syslog_server_enable = sql.get_setting('syslog_server_enable') if syslog_server_enable is None or syslog_server_enable == 0: if service == 'nginx': - local_path_logs = sql.get_setting('nginx_path_error_logs') - commands = ["sudo cat %s| awk '$2>\"%s:00\" && $2<\"%s:00\"' |tail -%s %s %s" % (local_path_logs, date, date1, rows, grep_act, exgrep_act)] + local_path_logs = sql.get_setting('nginx_path_logs') + commands = ["sudo cat %s/%s |tail -%s %s %s" % (local_path_logs, log_file, rows, grep_act, exgrep_act)] else: local_path_logs = sql.get_setting('haproxy_path_logs') - commands = ["sudo cat %s| awk '$3>\"%s:00\" && $3<\"%s:00\"' |tail -%s %s %s" % (local_path_logs, date, date1, rows, grep_act, exgrep_act)] + commands = ["sudo cat %s/%s| awk '$3>\"%s:00\" && $3<\"%s:00\"' |tail -%s %s %s" % (local_path_logs, log_file, date, date1, rows, grep_act, exgrep_act)] syslog_server = serv else: commands = ["sudo cat /var/log/%s/syslog.log | sed '/ %s:00/,/ %s:00/! d' |tail -%s %s %s %s" % (serv, date, date1, rows, grep_act, grep, exgrep_act)] @@ -1399,15 +1435,25 @@ def get_files(dir=get_config_var('configs', 'haproxy_save_configs_dir'), format= def get_remote_files(server_ip: str, config_dir: str, file_format: str): - if 'nginx' not in config_dir: - return 'error: The path must contain the name of the service. Check it in Roxy-WI settings' - if config_dir[-1] != '/': - config_dir += '/' - commands = ['ls ' + config_dir + '*.' + file_format] + config_dir = return_nice_path(config_dir) + if file_format == 'conf': + commands = ['ls ' + config_dir + '*/*.' + file_format] + else: + commands = ['ls ' + config_dir + '/*.' + file_format] config_files = ssh_command(server_ip, commands) return config_files + +def return_nice_path(return_path: str) -> str: + if 'nginx' not in return_path and 'haproxy' not in return_path: + return 'error: The path must contain the name of the service. Check it in Roxy-WI settings' + if return_path[-1] != '/': + return_path += '/' + + return return_path + + def get_key(item): return item[0] @@ -1693,6 +1739,26 @@ def get_system_info(server_ip: str) -> bool: except Exception: pass + try: + if b['class'] == 'storage': + for p, pval in b.items(): + if isinstance(pval, list): + for disks_info in pval: + for volume_info in disks_info['children']: + if isinstance(volume_info['logicalname'], list): + volume_name = volume_info['logicalname'][0] + mount_point = volume_info['logicalname'][1] + size = round(volume_info['capacity'] / 1073741824) + size = str(size) + 'Gb' + fs = volume_info['configuration']['mount.fstype'] + state = volume_info['configuration']['state'] + disks[volume_name] = {'mount_point': mount_point, + 'size': size, + 'fs': fs, + 'state': state} + except Exception: + pass + try: if b['class'] == 'bridge': if 'children' in b: @@ -1764,6 +1830,20 @@ def get_system_info(server_ip: str) -> bool: try: for q in y['children']: for o in q['children']: + try: + volume_name = o['logicalname'] + mount_point = '' + size = round(o['size'] / 1073741824) + size = str(size) + 'Gb' + fs = '' + state = '' + disks[volume_name] = { + 'mount_point': mount_point, + 'size': size, + 'fs': fs, + 'state': state} + except Exception: + pass for w in o['children']: try: if isinstance(w['logicalname'], list): diff --git a/app/logs.py b/app/logs.py index f182c0e6..7ee78713 100644 --- a/app/logs.py +++ b/app/logs.py @@ -39,6 +39,9 @@ except Exception: if service == 'nginx': if funct.check_login(service=2): title = "Nginx`s logs" +elif waf == '1': + if funct.check_login(service=1): + title = "WAF logs" else: if funct.check_login(service=1): title = "HAProxy`s logs" diff --git a/app/options.py b/app/options.py index 6efb02a0..3975dfb3 100644 --- a/app/options.py +++ b/app/options.py @@ -479,36 +479,36 @@ if act == "overviewHapserverBackends": if service == 'haproxy': configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir') format_file = 'cfg' - elif service == 'nginx': - configs_dir = funct.get_config_var('configs', 'nginx_save_configs_dir') - format_file = 'conf' elif service == 'keepalived': configs_dir = funct.get_config_var('configs', 'kp_save_configs_dir') format_file = 'conf' - try: - sections = funct.get_sections(configs_dir + funct.get_files(dir=configs_dir, format=format_file)[0], service=service) - except Exception as e: - funct.logging('localhost', str(e), haproxywi=1) + if service != 'nginx': + try: + sections = funct.get_sections(configs_dir + funct.get_files(dir=configs_dir, format=format_file)[0], service=service) + except Exception as e: + funct.logging('localhost', str(e), haproxywi=1) - try: - cfg = configs_dir + serv + "-" + funct.get_data('config') + '.' + format_file - except Exception as e: - funct.logging('localhost', ' Cannot generate a cfg path ' + str(e), haproxywi=1) - try: - if service == 'nginx': - error = funct.get_config(serv, cfg, nginx=1) - elif service == 'keepalived': - error = funct.get_config(serv, cfg, keepalived=1) - else: - error = funct.get_config(serv, cfg) - except Exception as e: - funct.logging('localhost', ' Cannot download a config ' + str(e), haproxywi=1) - try: - sections = funct.get_sections(cfg, service=service) - except Exception as e: - funct.logging('localhost', ' Cannot get sections from config file ' + str(e), haproxywi=1) - sections = 'Cannot get backends' + try: + cfg = configs_dir + serv + "-" + funct.get_data('config') + '.' + format_file + except Exception as e: + funct.logging('localhost', ' Cannot generate a cfg path ' + str(e), haproxywi=1) + try: + if service == 'nginx': + error = funct.get_config(serv, cfg, nginx=1) + elif service == 'keepalived': + error = funct.get_config(serv, cfg, keepalived=1) + else: + error = funct.get_config(serv, cfg) + except Exception as e: + funct.logging('localhost', ' Cannot download a config ' + str(e), haproxywi=1) + try: + sections = funct.get_sections(cfg, service=service) + except Exception as e: + funct.logging('localhost', ' Cannot get sections from config file ' + str(e), haproxywi=1) + sections = 'Cannot get backends' + else: + sections = funct.get_remote_sections(serv, service) template = template.render(backends=sections, serv=serv, service=service) print(template) @@ -1203,6 +1203,20 @@ if act == 'configShowFiles': template = template.render(serv=serv, return_files=return_files, config_file_name=config_file_name, path_dir=nginx_config_dir) print(template) +if act == 'showRemoteLogFiles': + service = form.getvalue('service') + log_path = sql.get_setting(service+'_path_logs') + return_files = funct.get_remote_files(serv, log_path, 'log') + if 'error: ' in return_files: + print(return_files) + sys.exit() + from jinja2 import Environment, FileSystemLoader + + env = Environment(loader=FileSystemLoader('templates/'), autoescape=True) + template = env.get_template('ajax/show_log_files.html') + template = template.render(serv=serv, return_files=return_files, path_dir=log_path) + print(template) + if form.getvalue('master'): master = form.getvalue('master') slave = form.getvalue('slave') @@ -3113,7 +3127,7 @@ if form.getvalue('awsvars') or form.getvalue('awseditvars'): print('error: ' + stderr) else: print('ok') - + if form.getvalue('dovars') or form.getvalue('doeditvars'): if form.getvalue('dovars'): dovars = form.getvalue('dovars') @@ -3172,7 +3186,7 @@ if form.getvalue('dovalidate') or form.getvalue('doeditvalidate'): print('error: ' + stderr) else: print('ok') - + if form.getvalue('doworkspace'): workspace = form.getvalue('doworkspace') group = form.getvalue('do_create_group') @@ -3335,7 +3349,7 @@ if form.getvalue('awseditworkspace'): output, stderr = funct.subprocess_execute(cmd) except Exception as e: print('error: ' +str(e)) - + if stderr != '': stderr = stderr.strip() stderr = repr(stderr) @@ -4022,3 +4036,19 @@ if act == 'updateSystemInfo': print(template) else: print('error: Cannot update server info') + +if act == 'findInConfigs': + server_ip = serv + server_ip = funct.is_ip_or_dns(server_ip) + finding_words = form.getvalue('words') + service = form.getvalue('service') + log_path = sql.get_setting(service + '_dir') + log_path = funct.return_nice_path(log_path) + commands = ['sudo grep "%s" %s* -C 2 -Rn' % (finding_words, log_path)] + return_find = funct.ssh_command(server_ip, commands, raw='1') + return_find = funct.show_finding_in_config(return_find, grep=finding_words) + + if 'error: ' in return_find: + print(return_find) + sys.exit() + print(return_find) diff --git a/app/scripts/ansible/roles/haproxy/tasks/main.yml b/app/scripts/ansible/roles/haproxy/tasks/main.yml index 41d541df..d3b73d8e 100644 --- a/app/scripts/ansible/roles/haproxy/tasks/main.yml +++ b/app/scripts/ansible/roles/haproxy/tasks/main.yml @@ -10,12 +10,12 @@ - name: populate service facts service_facts: -- include: logs.yml - - include: configure.yml - include: installation.yml +- include: logs.yml + - name: Add syn_flood tasks include: syn_flood.yml when: (SYN_FLOOD is defined) and (SYN_FLOOD|length > 0) diff --git a/app/scripts/ansible/roles/waf/tasks/main.yml b/app/scripts/ansible/roles/waf/tasks/main.yml index 27135e8c..520da501 100644 --- a/app/scripts/ansible/roles/waf/tasks/main.yml +++ b/app/scripts/ansible/roles/waf/tasks/main.yml @@ -40,6 +40,18 @@ http_proxy: "{{PROXY}}" https_proxy: "{{PROXY}}" + - name: install the el9 RPMS for HAProxy + yum: + name: + - yajl-devel + state: latest + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + - ansible_facts['distribution_major_version'] == '8' + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" + - name: install the common RPMS for HAProxy yum: name: diff --git a/app/templates/ajax/haproxyservers_backends.html b/app/templates/ajax/haproxyservers_backends.html index 5cd397d1..2252e8fa 100644 --- a/app/templates/ajax/haproxyservers_backends.html +++ b/app/templates/ajax/haproxyservers_backends.html @@ -6,11 +6,20 @@ {% if backends == 'Cannot get backends' %} {{backends}} {% else %} + {% if service == 'nginx' or service == 'apache' %} + {% set backends = backends.split('\n') %} + {% endif %} {% for b in backends %} {% if service == 'haproxy' %} {{b}} + {% elif service == 'nginx' or service == 'apache' %} + {% set full_file = b.split(' ')[0] | replace ('/', '92') %} + {% set full_file = full_file.replace(':', '') %} + + {{b.split(' ')[1] | replace(';', '')}} + {% else %} {{b}} {% endif %} diff --git a/app/templates/ajax/show_configs_files.html b/app/templates/ajax/show_configs_files.html index 2b31123e..f94617d1 100644 --- a/app/templates/ajax/show_configs_files.html +++ b/app/templates/ajax/show_configs_files.html @@ -1,11 +1,12 @@ {% from 'include/input_macros.html' import input %} -
-
+
+ {% if 'cannot access' not in return_files %} {% if config_file_name == '' %}

Config files from {{serv}}

{% endif %} +

- {% for file in return_files.split() %} {% if file == config_file_name %} @@ -18,7 +19,79 @@ {{ input('serv', type='hidden', value=serv) }} {{ input('open', type='hidden', value='open') }} - Open + Open + Add + Find

+ +
+ + {% else %} +
{{return_files}}
+ {% endif %} +
- \ No newline at end of file + + + diff --git a/app/templates/ajax/show_log_files.html b/app/templates/ajax/show_log_files.html new file mode 100644 index 00000000..96367ffe --- /dev/null +++ b/app/templates/ajax/show_log_files.html @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/app/templates/ajax/show_system_info.html b/app/templates/ajax/show_system_info.html index 7fea2cf7..9641541c 100644 --- a/app/templates/ajax/show_system_info.html +++ b/app/templates/ajax/show_system_info.html @@ -56,7 +56,7 @@ {{ram_info.size}}Gb - +
- - - + + diff --git a/app/templates/base.html b/app/templates/base.html index 79951c01..d7f51245 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -37,6 +37,8 @@ + + diff --git a/app/templates/config.html b/app/templates/config.html index 0c9aa79a..df4eaaad 100644 --- a/app/templates/config.html +++ b/app/templates/config.html @@ -1,9 +1,14 @@ {% extends "base.html" %} {% block content %} + + + + +
{% if selects|length == 0 %} @@ -86,10 +91,10 @@ } if (cur_url[1].split('&')[0] == 'service=haproxy' || cur_url[1].split('&')[0] == 'service=None') { var myCodeMirror = CodeMirror.fromTextArea(document.getElementById("config_text_area"), - {mode: "haproxy", lineNumbers: true, autocapitalize: true, autocorrect: true, spellcheck: true}); + {mode: "haproxy", lineNumbers: true, autocapitalize: true, autocorrect: true, spellcheck: true, autoCloseBrackets: true}); } else { var myCodeMirror = CodeMirror.fromTextArea(document.getElementById("config_text_area"), - {mode: "nginx", lineNumbers: true, autocapitalize: true, autocorrect: true, spellcheck: true}); + {mode: "nginx", lineNumbers: true, autocapitalize: true, autocorrect: true, spellcheck: true, autoCloseBrackets: true}); }
CPU diff --git a/app/templates/ajax/show_user_current_group.html b/app/templates/ajax/show_user_current_group.html index 11fd00b4..07f234f9 100644 --- a/app/templates/ajax/show_user_current_group.html +++ b/app/templates/ajax/show_user_current_group.html @@ -1,14 +1,12 @@ -
Current group - -
Current group + +