From af228c236ef35a57276654a482b9ea099f33f383 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Thu, 3 Aug 2023 09:56:25 +0300 Subject: [PATCH] v6.3.17.0 Changelog: https://roxy-wi.org/changelog#6_3_17 --- app/add.py | 9 +- app/create_db.py | 99 +------ app/modules/config/add.py | 129 ++++++++- app/modules/config/runtime.py | 8 +- app/modules/db/sql.py | 2 +- app/modules/roxywi/logs.py | 6 +- app/modules/roxywi/roxy.py | 11 + app/modules/service/installation.py | 5 +- app/modules/tools/checker.py | 60 ++++ app/options.py | 264 ++++++------------ .../ansible/roles/haproxy/tasks/configure.yml | 4 +- .../roles/haproxy/tasks/installation.yml | 2 +- .../roles/haproxy/templates/haproxy.cfg.j2 | 18 +- app/scripts/install_haproxy.sh | 29 +- app/templates/add.html | 47 +++- app/templates/languages/en.html | 6 +- app/templates/languages/fr.html | 3 +- app/templates/languages/pt-br.html | 6 +- app/templates/languages/ru.html | 3 +- app/templates/servers.html | 1 + inc/add.js | 169 ++++++++++- inc/overview-6.3.9.js | 29 +- inc/script-6.3.9.js | 32 ++- inc/users.js | 8 +- 24 files changed, 592 insertions(+), 358 deletions(-) create mode 100644 app/modules/tools/checker.py diff --git a/app/add.py b/app/add.py index a7157b3c..f45b4b57 100644 --- a/app/add.py +++ b/app/add.py @@ -45,9 +45,9 @@ if all(v is None for v in [ print(str(e)) lib_path = get_config_var.get_config_var('main', 'lib_path') - dir = lib_path + "/" + sql.get_setting('lists_path') - white_dir = lib_path + "/" + sql.get_setting('lists_path') + "/" + user_group + "/white" - black_dir = lib_path + "/" + sql.get_setting('lists_path') + "/" + user_group + "/black" + dir = lib_path + "/lists" + white_dir = lib_path + "/lists/" + user_group + "/white" + black_dir = lib_path + "/lists/" + user_group + "/black" if not os.path.exists(dir): os.makedirs(dir) @@ -60,12 +60,13 @@ if all(v is None for v in [ white_lists = roxywi_common.get_files(folder=white_dir, file_format="lst") black_lists = roxywi_common.get_files(folder=black_dir, file_format="lst") + maps = roxywi_common.get_files(folder=f'{lib_path}/maps/{user_group}', file_format="map") template = template.render( h2=1, role=user_params['role'], user=user_params['user'], selects=user_params['servers'], add=form.getvalue('add'), conf_add=form.getvalue('conf'), group=user_group, options=sql.select_options(), saved_servers=sql.select_saved_servers(), white_lists=white_lists, black_lists=black_lists, user_services=user_params['user_services'], token=user_params['token'], - lang=user_params['lang'] + lang=user_params['lang'], maps=maps ) print(template) diff --git a/app/create_db.py b/app/create_db.py index 098ad81c..8f50ff97 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -22,9 +22,6 @@ def default_values(): {'param': 'ssl_local_path', 'value': 'certs', 'section': 'main', 'desc': 'Path to the directory with the saved local SSL certificates. The value of this parameter is ' 'specified as a relative path beginning with $HOME_ROXY_WI/app/', 'group': '1'}, - {'param': 'lists_path', 'value': 'lists', 'section': 'main', - 'desc': 'Path to the black and the wild list. The value of this parameter should be specified as a relative path beginning with $HOME_ROXY-WI', - 'group': '1'}, {'param': 'maxmind_key', 'value': '', 'section': 'main', 'desc': 'License key for downloading GeoIP DB. You can create it on maxmind.com', 'group': '1'}, {'param': 'haproxy_path_logs', 'value': '/var/log/haproxy/', 'section': 'haproxy', 'desc': 'The path for HAProxy logs', 'group': '1'}, {'param': 'syslog_server_enable', 'value': '0', 'section': 'logs', 'desc': 'Enable getting logs from a syslog server', 'group': '1'}, @@ -633,85 +630,6 @@ def update_db_v_4_3_0(): print("An error occurred:", e) -def update_db_v_6_0(): - cursor = conn.cursor() - sql = list() - sql.append("alter table servers add column apache integer default 0") - sql.append("alter table servers add column apache_active integer default 0") - sql.append("alter table servers add column apache_alert integer default 0") - sql.append("alter table servers add column apache_metrics integer default 0") - for i in sql: - try: - cursor.execute(i) - except Exception: - pass - else: - print('Updating... DB has been updated to version 6.0.0.0') - - -def update_db_v_6_0_1(): - query = Groups.update(name='Default').where(Groups.group_id == '1') - try: - query.execute() - except Exception as e: - print("An error occurred:", e) - else: - print("Updating... DB has been updated to version 6.0.0.0-1") - - -def update_db_v_6_1_0(): - for service_id in range(1, 5): - try: - servers_id = Server.select(Server.server_id).where(Server.type_ip == 0).execute() - for server_id in servers_id: - CheckerSetting.insert( - server_id=server_id, service_id=service_id - ).on_conflict_ignore().execute() - except Exception as e: - if e.args[0] == 'duplicate column name: haproxy' or str(e) == '(1060, "Duplicate column name \'haproxy\'")': - print('Updating... go to version 6.1.0') - else: - print("An error occurred:", e) - - -def update_db_v_6_1_3(): - if mysql_enable == '1': - cursor = conn.cursor() - sql = list() - sql.append("ALTER TABLE `waf_rules` ADD COLUMN service VARCHAR ( 64 ) DEFAULT 'haproxy'") - sql.append("ALTER TABLE `waf_rules` drop CONSTRAINT serv") - sql.append("ALTER TABLE `waf_rules` ADD CONSTRAINT UNIQUE (serv, rule_name, service)") - for i in sql: - try: - cursor.execute(i) - except Exception: - pass - else: - print('Updating... DB has been updated to version 6.1.3.0') - else: - pass - - -def update_db_v_6_1_4(): - servers = Server.select() - services = Services.select() - for server in servers: - for service in services: - settings = ('restart', 'dockerized', 'haproxy_enterprise') - for setting in settings: - if service.slug == 'keepalived': - continue - if service.slug != 'haproxy' and setting == 'haproxy_enterprise': - continue - set_value = 0 - try: - ServiceSetting.insert( - server_id=server.server_id, service=service.slug, setting=setting, value=set_value - ).on_conflict_ignore().execute() - except Exception: - pass - - def update_db_v_6_2_1(): try: Setting.update(section='main').where(Setting.param == 'maxmind_key').execute() @@ -914,9 +832,18 @@ def update_db_v_6_3_13_5(): print("An error occurred:", e) +def update_db_v_6_3_17(): + try: + Setting.delete().where(Setting.param == 'lists_path').execute() + except Exception as e: + print("An error occurred:", e) + else: + print("Updating... DB has been updated to version 6.3.17") + + def update_ver(): try: - Version.update(version='6.3.16.0').execute() + Version.update(version='6.3.17.0').execute() except Exception: print('Cannot update version') @@ -934,11 +861,6 @@ def update_all(): if check_ver() is None: update_db_v_3_4_5_22() update_db_v_4_3_0() - update_db_v_6_0() - update_db_v_6_0_1() - update_db_v_6_1_0() - update_db_v_6_1_3() - update_db_v_6_1_4() update_db_v_6_2_1() update_db_v_6_3_4() update_db_v_6_3_5() @@ -953,6 +875,7 @@ def update_all(): update_db_v_6_3_13_3() update_db_v_6_3_13_4() update_db_v_6_3_13_5() + update_db_v_6_3_17() update_ver() diff --git a/app/modules/config/add.py b/app/modules/config/add.py index ee28b94f..5acbcc29 100644 --- a/app/modules/config/add.py +++ b/app/modules/config/add.py @@ -52,7 +52,7 @@ def show_userlist(server_ip: str) -> None: def get_bwlist(color: str, group: str, list_name: str) -> None: lib_path = get_config.get_config_var('main', 'lib_path') - list_path = f"{lib_path}/{sql.get_setting('lists_path')}/{group}/{color}/{list_name}" + list_path = f"{lib_path}/lists/{group}/{color}/{list_name}" try: with open(list_path, 'r') as f: @@ -63,7 +63,7 @@ def get_bwlist(color: str, group: str, list_name: str) -> None: def get_bwlists_for_autocomplete(color: str, group: str) -> None: lib_path = get_config.get_config_var('main', 'lib_path') - list_path = f"{lib_path}/{sql.get_setting('lists_path')}/{group}/{color}" + list_path = f"{lib_path}/lists/{group}/{color}" lists = roxywi_common.get_files(list_path, "lst") for line in lists: @@ -73,7 +73,7 @@ def get_bwlists_for_autocomplete(color: str, group: str) -> None: def create_bwlist(server_ip: str, list_name: str, color: str, group: str) -> None: lib_path = get_config.get_config_var('main', 'lib_path') list_name = f"{list_name.split('.')[0]}.lst" - list_path = f"{lib_path}/{sql.get_setting('lists_path')}/{group}/{color}/{list_name}" + list_path = f"{lib_path}/lists/{group}/{color}/{list_name}" try: open(list_path, 'a').close() print('success: ') @@ -87,7 +87,7 @@ def create_bwlist(server_ip: str, list_name: str, color: str, group: str) -> Non def save_bwlist(list_name: str, list_con: str, color: str, group: str, server_ip: str, action: str) -> None: lib_path = get_config.get_config_var('main', 'lib_path') - list_path = f"{lib_path}/{sql.get_setting('lists_path')}/{group}/{color}/{list_name}" + list_path = f"{lib_path}/lists/{group}/{color}/{list_name}" try: with open(list_path, "w") as file: file.write(list_con) @@ -139,7 +139,7 @@ def save_bwlist(list_name: str, list_con: str, color: str, group: str, server_ip def delete_bwlist(list_name: str, color: str, group: str, server_ip: str) -> None: servers = [] lib_path = get_config.get_config_var('main', 'lib_path') - list_path = f"{lib_path}/{sql.get_setting('lists_path')}/{group}/{color}/{list_name}" + list_path = f"{lib_path}/lists/{group}/{color}/{list_name}" path = f"{sql.get_setting('haproxy_dir')}/{color}" try: @@ -172,6 +172,125 @@ def delete_bwlist(list_name: str, color: str, group: str, server_ip: str) -> Non pass +def edit_map(map_name: str, group: str) -> None: + lib_path = get_config.get_config_var('main', 'lib_path') + list_path = f"{lib_path}/maps/{group}/{map_name}" + + try: + with open(list_path, 'r') as f: + print(f.read()) + except IOError as e: + print(f"error: Cannot read {map_name} list: {e}") + + +def create_map(server_ip: str, map_name: str, group: str) -> None: + lib_path = get_config.get_config_var('main', 'lib_path') + map_name = f"{map_name.split('.')[0]}.map" + map_path = f'{lib_path}/maps/{group}/' + full_path = f'{map_path}/{map_name}' + + try: + server_mod.subprocess_execute(f'mkdir -p {map_path}') + except Exception as e: + assert Exception(f'error: cannot create a local folder for maps: {e}') + try: + open(full_path, 'a').close() + print('success: ') + try: + roxywi_common.logging(server_ip, f'A new map {map_name} has been created', roxywi=1, login=1) + except Exception: + pass + except IOError as e: + assert Exception(f'error: Cannot create a new {map_name} map. {e}, ') + + +def save_map(map_name: str, list_con: str, group: str, server_ip: str, action: str) -> None: + lib_path = get_config.get_config_var('main', 'lib_path') + map_path = f"{lib_path}/maps/{group}/{map_name}" + try: + with open(map_path, "w") as file: + file.write(list_con) + except IOError as e: + print(f'error: Cannot save {map_name} list. {e}') + + path = sql.get_setting('haproxy_dir') + "/maps" + servers = [] + + if server_ip != 'all': + servers.append(server_ip) + + masters = sql.is_master(server_ip) + for master in masters: + if master[0] is not None: + servers.append(master[0]) + else: + server = roxywi_common.get_dick_permit() + for s in server: + servers.append(s[2]) + + for serv in servers: + server_mod.ssh_command(serv, [f"sudo mkdir {path}"]) + server_mod.ssh_command(serv, [f"sudo chown $(whoami) {path}"]) + error = config_mod.upload(serv, f'{path}/{map_name}', map_path, dir='fullpath') + + if error: + print(f'error: Upload fail: {error} , ') + else: + print(f'success: Edited {map_name} map was uploaded to {serv} , ') + try: + roxywi_common.logging(serv, f'Has been edited the map {map_name}', roxywi=1, login=1) + except Exception: + pass + + server_id = sql.select_server_id_by_ip(server_ip=serv) + haproxy_enterprise = sql.select_service_setting(server_id, 'haproxy', 'haproxy_enterprise') + if haproxy_enterprise == '1': + haproxy_service_name = "hapee-2.0-lb" + else: + haproxy_service_name = "haproxy" + + if action == 'restart': + server_mod.ssh_command(serv, [f"sudo systemctl restart {haproxy_service_name}"]) + elif action == 'reload': + server_mod.ssh_command(serv, [f"sudo systemctl reload {haproxy_service_name}"]) + + +def delete_map(map_name: str, group: str, server_ip: str) -> None: + servers = [] + lib_path = get_config.get_config_var('main', 'lib_path') + list_path = f"{lib_path}/maps/{group}/{map_name}" + path = f"{sql.get_setting('haproxy_dir')}/maps" + + try: + os.remove(list_path) + except IOError as e: + print(f'error: Cannot delete {map_name} map from Roxy-WI server. {e} , ') + + if server_ip != 'all': + servers.append(server_ip) + + masters = sql.is_master(server_ip) + for master in masters: + if master[0] is not None: + servers.append(master[0]) + else: + server = roxywi_common.get_dick_permit() + for s in server: + servers.append(s[2]) + + for serv in servers: + error = server_mod.ssh_command(serv, [f"sudo rm {path}/{map_name}"], return_err=1) + + if error: + print(f'error: Deleting fail: {error} , ') + else: + print(f'success: the {map_name} map has been deleted on {serv} , ') + try: + roxywi_common.logging(serv, f'has been deleted the {map_name} map', roxywi=1, login=1) + except Exception: + pass + + def create_saved_option(option: str, group: int) -> None: if sql.insert_new_option(option, group): from jinja2 import Environment, FileSystemLoader diff --git a/app/modules/config/runtime.py b/app/modules/config/runtime.py index b85b06ad..99365887 100644 --- a/app/modules/config/runtime.py +++ b/app/modules/config/runtime.py @@ -349,15 +349,14 @@ def show_lists() -> None: def delete_ip_from_list() -> None: haproxy_sock_port = sql.get_setting('haproxy_sock_port') - lists_path = sql.get_setting('lists_path') lib_path = get_config_var.get_config_var('main', 'lib_path') ip_id = common.checkAjaxInput(form.getvalue('list_ip_id_for_delete')) ip = common.is_ip_or_dns(form.getvalue('list_ip_for_delete')) list_id = common.checkAjaxInput(form.getvalue('list_id_for_delete')) list_name = common.checkAjaxInput(form.getvalue('list_name')) user_group = roxywi_common.get_user_group(id=1) - cmd = f"sed -i 's!{ip}$!!' {lib_path}/{lists_path}/{user_group}/{list_name}" - cmd1 = f"sed -i '/^$/d' {lib_path}/{lists_path}/{user_group}/{list_name}" + cmd = f"sed -i 's!{ip}$!!' {lib_path}/lists/{user_group}/{list_name}" + cmd1 = f"sed -i '/^$/d' {lib_path}/lists/{user_group}/{list_name}" output, stderr = server_mod.subprocess_execute(cmd) output1, stderr1 = server_mod.subprocess_execute(cmd1) if output: @@ -381,7 +380,6 @@ def delete_ip_from_list() -> None: def add_ip_to_list() -> None: haproxy_sock_port = sql.get_setting('haproxy_sock_port') - lists_path = sql.get_setting('lists_path') lib_path = get_config_var.get_config_var('main', 'lib_path') ip = form.getvalue('list_ip_for_add') ip = ip.strip() @@ -397,7 +395,7 @@ def add_ip_to_list() -> None: print(f'error: {stderr[0]}') if 'is not a valid IPv4 or IPv6 address' not in output[0]: - cmd = f'echo "{ip}" >> {lib_path}/{lists_path}/{user_group}/{list_name}' + cmd = f'echo "{ip}" >> {lib_path}/lists/{user_group}/{list_name}' output, stderr = server_mod.subprocess_execute(cmd) if output: print(f'error: {output}') diff --git a/app/modules/db/sql.py b/app/modules/db/sql.py index 828a6bc4..244fc9a2 100755 --- a/app/modules/db/sql.py +++ b/app/modules/db/sql.py @@ -26,7 +26,7 @@ def get_setting(param, **kwargs): except Exception: pass - if user_group == '' or param in ('lists_path', 'ssl_local_path', 'proxy'): + if user_group == '' or param in ('ssl_local_path', 'proxy'): user_group = 1 if kwargs.get('all'): diff --git a/app/modules/roxywi/logs.py b/app/modules/roxywi/logs.py index eb247389..3490484b 100644 --- a/app/modules/roxywi/logs.py +++ b/app/modules/roxywi/logs.py @@ -55,12 +55,12 @@ def show_log(stdout, **kwargs): def show_roxy_log( serv, rows='10', waf='0', grep=None, hour='00', - minut='00', hour1='24', minut1='00', service='haproxy', **kwargs + minute='00', hour1='24', minute1='00', service='haproxy', **kwargs ) -> str: exgrep = form.getvalue('exgrep') log_file = form.getvalue('file') - date = checkAjaxInput(hour) + ':' + checkAjaxInput(minut) - date1 = checkAjaxInput(hour1) + ':' + checkAjaxInput(minut1) + date = checkAjaxInput(hour) + ':' + checkAjaxInput(minute) + date1 = checkAjaxInput(hour1) + ':' + checkAjaxInput(minute1) rows = checkAjaxInput(rows) waf = checkAjaxInput(waf) cmd = '' diff --git a/app/modules/roxywi/roxy.py b/app/modules/roxywi/roxy.py index 1ba3406d..e40ef724 100644 --- a/app/modules/roxywi/roxy.py +++ b/app/modules/roxywi/roxy.py @@ -21,6 +21,17 @@ def is_docker() -> bool: def update_roxy_wi(service): restart_service = '' + services = ['roxy-wi-checker', + 'roxy-wi', + 'roxy-wi-keep_alive', + 'roxy-wi-smon', + 'roxy-wi-metrics', + 'roxy-wi-portscanner', + 'roxy-wi-socket', + 'roxy-wi-prometheus-exporter'] + + if service not in services: + raise Exception(f'error: {service} is not part of Roxy-WI') if distro.id() == 'ubuntu': try: diff --git a/app/modules/service/installation.py b/app/modules/service/installation.py index 86deb4a6..e261ad2c 100644 --- a/app/modules/service/installation.py +++ b/app/modules/service/installation.py @@ -49,6 +49,9 @@ def install_haproxy(server_ip: str, api=0, **kwargs): haproxy_ver = kwargs.get('hapver') server_for_installing = kwargs.get('server') docker = kwargs.get('docker') + m_or_s = kwargs.get('m_or_s') + master = kwargs.get('master') + slave = kwargs.get('slave') proxy_serv = '' ssh_settings = return_ssh_keys_path(server_ip) full_path = '/var/www/haproxy-wi/app' @@ -68,7 +71,7 @@ def install_haproxy(server_ip: str, api=0, **kwargs): f"STAT_FILE={server_state_file} DOCKER={docker} SSH_PORT={ssh_settings['port']} STATS_USER={stats_user} " f"CONT_NAME={container_name} HAP_DIR={haproxy_dir} STATS_PASS='{stats_password}' HAPVER={haproxy_ver} " f"SYN_FLOOD={syn_flood_protect} HOST={server_ip} USER={ssh_settings['user']} PASS='{ssh_settings['password']}' " - f"KEY={ssh_settings['key']}" + f"M_OR_S={m_or_s} MASTER={master} SLAVE={slave} KEY={ssh_settings['key']}" ] if server_for_installing: diff --git a/app/modules/tools/checker.py b/app/modules/tools/checker.py new file mode 100644 index 00000000..e5b45499 --- /dev/null +++ b/app/modules/tools/checker.py @@ -0,0 +1,60 @@ +from jinja2 import Environment, FileSystemLoader + +import modules.db.sql as sql +import modules.roxywi.common as roxywi_common +import modules.roxywi.roxy as roxywi_mod + + +def load_checker(page: str) -> None: + groups = sql.select_groups() + services = roxywi_mod.get_services_status() + lang = roxywi_common.get_user_lang() + env = Environment(loader=FileSystemLoader('templates'), autoescape=True) + template = env.get_template('ajax/load_telegram.html') + keepalived_settings = '' + haproxy_settings = '' + apache_settings = '' + nginx_settings = '' + keepalived_servers = '' + haproxy_servers = '' + apache_servers = '' + nginx_servers = '' + telegrams = '' + slacks = '' + pds = '' + + try: + user_subscription = roxywi_common.return_user_status() + except Exception as e: + user_subscription = roxywi_common.return_unsubscribed_user_status() + roxywi_common.logging('Roxy-WI server', f'Cannot get a user plan: {e}', roxywi=1) + + if user_subscription['user_status']: + haproxy_settings = sql.select_checker_settings(1) + nginx_settings = sql.select_checker_settings(2) + keepalived_settings = sql.select_checker_settings(3) + apache_settings = sql.select_checker_settings(4) + if page == 'servers.py': + user_group = roxywi_common.get_user_group(id=1) + telegrams = sql.get_user_telegram_by_group(user_group) + slacks = sql.get_user_slack_by_group(user_group) + pds = sql.get_user_pd_by_group(user_group) + haproxy_servers = roxywi_common.get_dick_permit(haproxy=1, only_group=1) + nginx_servers = roxywi_common.get_dick_permit(nginx=1, only_group=1) + apache_servers = roxywi_common.get_dick_permit(apache=1, only_group=1) + keepalived_servers = roxywi_common.get_dick_permit(keepalived=1, only_group=1) + else: + telegrams = sql.select_telegram() + slacks = sql.select_slack() + pds = sql.select_pd() + haproxy_servers = roxywi_common.get_dick_permit(haproxy=1) + nginx_servers = roxywi_common.get_dick_permit(nginx=1) + apache_servers = roxywi_common.get_dick_permit(apache=1) + keepalived_servers = roxywi_common.get_dick_permit(keepalived=1) + + template = template.render(services=services, telegrams=telegrams, pds=pds, groups=groups, slacks=slacks, + user_status=user_subscription['user_status'], user_plan=user_subscription['user_plan'], + haproxy_servers=haproxy_servers, nginx_servers=nginx_servers, apache_servers=apache_servers, + keepalived_servers=keepalived_servers, haproxy_settings=haproxy_settings, nginx_settings=nginx_settings, + keepalived_settings=keepalived_settings, apache_settings=apache_settings, page=page, lang=lang) + print(template) diff --git a/app/options.py b/app/options.py index 4934f86c..1e5a8106 100644 --- a/app/options.py +++ b/app/options.py @@ -276,46 +276,32 @@ if form.getvalue('action'): if serv is not None and act == "stats": service_common.get_stat_page(serv, service) -if serv is not None and form.getvalue('show_log') is not None: +if serv is not None and any((form.getvalue('show_log'), form.getvalue('rows1'), form.getvalue('viewlogs'))): import modules.roxywi.logs as roxywi_logs + waf = 0 rows = form.getvalue('show_log') - waf = form.getvalue('waf') + service = service + + if form.getvalue('rows1'): + rows = form.getvalue('rows1') + service = 'apache_internal' + elif form.getvalue('show_log'): + waf = form.getvalue('waf') + elif form.getvalue('viewlogs'): + serv = form.getvalue('viewlogs') + rows = form.getvalue('rows') + service = 'internal' + grep = form.getvalue('grep') hour = form.getvalue('hour') - minut = form.getvalue('minut') + minute = form.getvalue('minut') hour1 = form.getvalue('hour1') - minut1 = form.getvalue('minut1') - out = roxywi_logs.show_roxy_log(serv, rows=rows, waf=waf, grep=grep, hour=hour, minut=minut, hour1=hour1, - minut1=minut1, service=service) - print(out) + minute1 = form.getvalue('minut1') -if serv is not None and form.getvalue('rows1') is not None: - import modules.roxywi.logs as roxywi_logs - - rows = form.getvalue('rows1') - grep = form.getvalue('grep') - hour = form.getvalue('hour') - minut = form.getvalue('minut') - hour1 = form.getvalue('hour1') - minut1 = form.getvalue('minut1') - out = roxywi_logs.show_roxy_log(serv, rows=rows, waf='0', grep=grep, hour=hour, minut=minut, hour1=hour1, - minut1=minut1, service='apache_internal') - print(out) - -if form.getvalue('viewlogs') is not None: - import modules.roxywi.logs as roxywi_logs - - viewlog = form.getvalue('viewlogs') - rows = form.getvalue('rows') - grep = form.getvalue('grep') - hour = form.getvalue('hour') - minut = form.getvalue('minut') - hour1 = form.getvalue('hour1') - minut1 = form.getvalue('minut1') if roxywi_common.check_user_group(): - out = roxywi_logs.show_roxy_log(serv=viewlog, rows=rows, waf='0', grep=grep, hour=hour, minut=minut, hour1=hour1, - minut1=minut1, service='internal') + out = roxywi_logs.show_roxy_log(serv=serv, rows=rows, waf=waf, grep=grep, hour=hour, minute=minute, hour1=hour1, + minute1=minute1, service=service) print(out) if serv is not None and act == "showMap": @@ -407,12 +393,12 @@ if form.getvalue('master_slave_hap'): if server == 'master': try: - service_mod.install_haproxy(master, server=server, docker=docker) + service_mod.install_haproxy(master, server=server, docker=docker, m_or_s='master', master=master, slave=slave) except Exception as e: print(f'{e}') elif server == 'slave': try: - service_mod.install_haproxy(slave, server=server, docker=docker) + service_mod.install_haproxy(slave, server=server, docker=docker, m_or_s='slave', master=master, slave=slave) except Exception as e: print(f'{e}') @@ -586,18 +572,12 @@ if form.getvalue('geoip_install'): if form.getvalue('update_roxy_wi'): import modules.roxywi.roxy as roxy - services = ['roxy-wi-checker', - 'roxy-wi', - 'roxy-wi-keep_alive', - 'roxy-wi-smon', - 'roxy-wi-metrics', - 'roxy-wi-portscanner', - 'roxy-wi-socket', - 'roxy-wi-prometheus-exporter'] - if service not in services: - print(f'error: {service} is not part of Roxy-WI') - sys.exit() - roxy.update_roxy_wi(service) + service = form.getvalue('service') + + try: + roxy.update_roxy_wi(service) + except Exception as e: + print(e) if form.getvalue('metrics_waf'): metrics_waf = common.checkAjaxInput(form.getvalue('metrics_waf')) @@ -717,6 +697,36 @@ if form.getvalue('get_lists'): color = common.checkAjaxInput(form.getvalue('color')) add_mod.get_bwlists_for_autocomplete(color, group) +if form.getvalue('edit_map'): + group = common.checkAjaxInput(form.getvalue('group')) + map_name = common.checkAjaxInput(form.getvalue('edit_map')) + + add_mod.edit_map(map_name, group) + +if form.getvalue('map_create'): + map_name = common.checkAjaxInput(form.getvalue('map_create')) + group = common.checkAjaxInput(form.getvalue('group')) + + try: + add_mod.create_map(serv, map_name, group) + except Exception as e: + print(e) + +if form.getvalue('map_save'): + group = common.checkAjaxInput(form.getvalue('group')) + map_save = common.checkAjaxInput(form.getvalue('map_save')) + content = form.getvalue('content') + action = common.checkAjaxInput(form.getvalue('map_restart')) + + add_mod.save_map(map_save, content, group, serv, action) + +if form.getvalue('map_delete'): + map_name = common.checkAjaxInput(form.getvalue('map_delete')) + group = common.checkAjaxInput( form.getvalue('group')) + server_id = common.checkAjaxInput( form.getvalue('serv')) + + add_mod.delete_map(map_name, group, server_id) + if form.getvalue('get_ldap_email'): import modules.roxywi.user as roxywi_user @@ -1347,54 +1357,9 @@ if form.getvalue('loadservices'): print(template) if form.getvalue('loadchecker'): - from modules.roxywi.roxy import get_services_status + import modules.tools.checker as checker_mod - lang = roxywi_common.get_user_lang() - env = Environment(loader=FileSystemLoader('templates'), autoescape=True) - template = env.get_template('ajax/load_telegram.html') - services = get_services_status() - groups = sql.select_groups() - page = form.getvalue('page') - - try: - user_subscription = roxywi_common.return_user_status() - except Exception as e: - user_subscription = roxywi_common.return_unsubscribed_user_status() - roxywi_common.logging('Roxy-WI server', f'Cannot get a user plan: {e}', roxywi=1) - - if user_subscription['user_status']: - haproxy_settings = sql.select_checker_settings(1) - nginx_settings = sql.select_checker_settings(2) - keepalived_settings = sql.select_checker_settings(3) - apache_settings = sql.select_checker_settings(4) - if page == 'servers.py': - user_group = roxywi_common.get_user_group(id=1) - telegrams = sql.get_user_telegram_by_group(user_group) - slacks = sql.get_user_slack_by_group(user_group) - pds = sql.get_user_pd_by_group(user_group) - haproxy_servers = roxywi_common.get_dick_permit(haproxy=1, only_group=1) - nginx_servers = roxywi_common.get_dick_permit(nginx=1, only_group=1) - apache_servers = roxywi_common.get_dick_permit(apache=1, only_group=1) - keepalived_servers = roxywi_common.get_dick_permit(keepalived=1, only_group=1) - else: - telegrams = sql.select_telegram() - slacks = sql.select_slack() - pds = sql.select_pd() - haproxy_servers = roxywi_common.get_dick_permit(haproxy=1) - nginx_servers = roxywi_common.get_dick_permit(nginx=1) - apache_servers = roxywi_common.get_dick_permit(apache=1) - keepalived_servers = roxywi_common.get_dick_permit(keepalived=1) - else: - telegrams = '' - slacks = '' - pds = '' - - template = template.render(services=services, telegrams=telegrams, pds=pds, groups=groups, slacks=slacks, - user_status=user_subscription['user_status'], user_plan=user_subscription['user_plan'], - haproxy_servers=haproxy_servers, nginx_servers=nginx_servers, apache_servers=apache_servers, - keepalived_servers=keepalived_servers, haproxy_settings=haproxy_settings, nginx_settings=nginx_settings, - keepalived_settings=keepalived_settings, apache_settings=apache_settings, page=page, lang=lang) - print(template) + checker_mod.load_checker() if form.getvalue('load_update_hapwi'): import modules.roxywi.roxy as roxy @@ -1541,13 +1506,13 @@ if form.getvalue('serverSettingsSave') is not None: server_id = common.checkAjaxInput(form.getvalue('serverSettingsSave')) service = common.checkAjaxInput(form.getvalue('serverSettingsService')) haproxy_enterprise = common.checkAjaxInput(form.getvalue('serverSettingsEnterprise')) - haproxy_dockerized = common.checkAjaxInput(form.getvalue('serverSettingshaproxy_dockerized')) - nginx_dockerized = common.checkAjaxInput(form.getvalue('serverSettingsnginx_dockerized')) - apache_dockerized = common.checkAjaxInput(form.getvalue('serverSettingsapache_dockerized')) - haproxy_restart = common.checkAjaxInput(form.getvalue('serverSettingsHaproxyrestart')) - nginx_restart = common.checkAjaxInput(form.getvalue('serverSettingsNginxrestart')) - apache_restart = common.checkAjaxInput(form.getvalue('serverSettingsApache_restart')) + service_dockerized = common.checkAjaxInput(form.getvalue('serverSettingsDockerized')) + service_restart = common.checkAjaxInput(form.getvalue('serverSettingsRestart')) server_ip = sql.select_server_ip_by_id(server_id) + service_docker = f'Service {service.title()} has been flagged as a dockerized' + service_systemd = f'Service {service.title()} has been flagged as a system service' + disable_restart = f'Restart option is disabled for {service.title()} service' + enable_restart = f'Restart option is disabled for {service.title()} service' if service == 'haproxy': if sql.insert_or_update_service_setting(server_id, service, 'haproxy_enterprise', haproxy_enterprise): @@ -1558,58 +1523,20 @@ if form.getvalue('serverSettingsSave') is not None: else: roxywi_common.logging(server_ip, 'Service has been flagged as a community version', roxywi=1, login=1, keep_history=1, service=service) - if sql.insert_or_update_service_setting(server_id, service, 'dockerized', haproxy_dockerized): - print('Ok') - if haproxy_dockerized == '1': - roxywi_common.logging(server_ip, 'Service has been flagged as a dockerized', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Service has been flagged as a system service', roxywi=1, login=1, - keep_history=1, service=service) - if sql.insert_or_update_service_setting(server_id, service, 'restart', haproxy_restart): - print('Ok') - if haproxy_restart == '1': - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) - if service == 'nginx': - if sql.insert_or_update_service_setting(server_id, service, 'dockerized', nginx_dockerized): - print('Ok') - if nginx_dockerized: - roxywi_common.logging(server_ip, 'Service has been flagged as a dockerized', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Service has been flagged as a system service', roxywi=1, login=1, - keep_history=1, service=service) - if sql.insert_or_update_service_setting(server_id, service, 'restart', nginx_restart): - print('Ok') - if nginx_restart == '1': - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) + if sql.insert_or_update_service_setting(server_id, service, 'dockerized', service_dockerized): + print('Ok') + if service_dockerized == '1': + roxywi_common.logging(server_ip, service_docker, roxywi=1, login=1, keep_history=1, service=service) + else: + roxywi_common.logging(server_ip, service_systemd, roxywi=1, login=1, keep_history=1, service=service) - if service == 'apache': - if sql.insert_or_update_service_setting(server_id, service, 'dockerized', apache_dockerized): - print('Ok') - if apache_dockerized: - roxywi_common.logging(server_ip, 'Service has been flagged as a dockerized', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Service has been flagged as a system service', roxywi=1, login=1, - keep_history=1, service=service) - if sql.insert_or_update_service_setting(server_id, service, 'restart', apache_restart): - print('Ok') - if apache_restart == '1': - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) - else: - roxywi_common.logging(server_ip, 'Restart option is disabled for this service', roxywi=1, login=1, - keep_history=1, service=service) + if sql.insert_or_update_service_setting(server_id, service, 'restart', service_restart): + print('Ok') + if service_restart == '1': + roxywi_common.logging(server_ip, disable_restart, roxywi=1, login=1, keep_history=1, service=service) + else: + roxywi_common.logging(server_ip, enable_restart, roxywi=1, login=1, keep_history=1, service=service) if act == 'showListOfVersion': config_mod.list_of_versions(serv, service) @@ -1655,46 +1582,19 @@ if form.getvalue('show_sub_ovw'): roxywi_overview.show_sub_ovw() if form.getvalue('updateHaproxyCheckerSettings'): - setting_id = form.getvalue('updateHaproxyCheckerSettings') - email = form.getvalue('email') - service_alert = form.getvalue('server') - backend_alert = form.getvalue('backend') - maxconn_alert = form.getvalue('maxconn') - telegram_id = form.getvalue('telegram_id') - slack_id = form.getvalue('slack_id') - pd_id = form.getvalue('pd_id') + import modules.tools.checker as checker_mod - if sql.update_haproxy_checker_settings(email, telegram_id, slack_id, pd_id, service_alert, backend_alert, maxconn_alert, setting_id): - print('ok') - else: - print('error: Cannot update Checker settings') + checker_mod.update_haproxy_settings() if form.getvalue('updateKeepalivedCheckerSettings'): - setting_id = form.getvalue('updateKeepalivedCheckerSettings') - email = form.getvalue('email') - service_alert = form.getvalue('server') - backend_alert = form.getvalue('backend') - telegram_id = form.getvalue('telegram_id') - slack_id = form.getvalue('slack_id') - pd_id = form.getvalue('pd_id') + import modules.tools.checker as checker_mod - if sql.update_keepalived_checker_settings(email, telegram_id, slack_id, pd_id, service_alert, backend_alert, setting_id): - print('ok') - else: - print('error: Cannot update Checker settings') + checker_mod.update_keepalived_settings() if form.getvalue('updateServiceCheckerSettings'): - setting_id = form.getvalue('updateServiceCheckerSettings') - email = form.getvalue('email') - service_alert = form.getvalue('server') - telegram_id = form.getvalue('telegram_id') - slack_id = form.getvalue('slack_id') - pd_id = form.getvalue('pd_id') + import modules.tools.checker as checker_mod - if sql.update_service_checker_settings(email, telegram_id, slack_id, pd_id, service_alert, setting_id): - print('ok') - else: - print('error: Cannot update Checker settings') + checker_mod.update_service_settings() if act == 'show_server_services': server_mod.show_server_services() diff --git a/app/scripts/ansible/roles/haproxy/tasks/configure.yml b/app/scripts/ansible/roles/haproxy/tasks/configure.yml index f9834470..07ce9aa1 100644 --- a/app/scripts/ansible/roles/haproxy/tasks/configure.yml +++ b/app/scripts/ansible/roles/haproxy/tasks/configure.yml @@ -46,7 +46,7 @@ - '"firewalld" in ansible_facts.packages' - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' - ansible_facts.services["firewalld.service"]['state'] == "running" - with_items: [ "{{ STAT_PORT }}", "{{ SOCK_PORT }}" ] + with_items: [ "{{ STAT_PORT }}", "{{ SOCK_PORT }}", "10000" ] - name: Open stat port for iptables @@ -56,7 +56,7 @@ jump: ACCEPT protocol: tcp ignore_errors: yes - with_items: [ "{{ STAT_PORT }}", "{{ SOCK_PORT }}" ] + with_items: [ "{{ STAT_PORT }}", "{{ SOCK_PORT }}", "10000" ] - name: Create the haproxy group group: diff --git a/app/scripts/ansible/roles/haproxy/tasks/installation.yml b/app/scripts/ansible/roles/haproxy/tasks/installation.yml index 16a8c71c..c6244e81 100644 --- a/app/scripts/ansible/roles/haproxy/tasks/installation.yml +++ b/app/scripts/ansible/roles/haproxy/tasks/installation.yml @@ -61,7 +61,7 @@ - socat - rsyslog state: latest - when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and ("'FAILED' in install_result1.stderr") + when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and (install_result.rc != 0 and install_result1.rc != 0) register: install_result retries: 5 until: install_result.rc == 0 diff --git a/app/scripts/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/app/scripts/ansible/roles/haproxy/templates/haproxy.cfg.j2 index 454b8130..e4a5178a 100644 --- a/app/scripts/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/app/scripts/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -38,9 +38,21 @@ listen stats stats realm HAProxy-04\ Statistics stats auth {{STATS_USER}}:{{STATS_PASS}} stats admin if TRUE - + +{% if M_OR_S != 'None' %} +peers default_peers + {% if M_OR_S == 'master' %} + peer {{ ansible_hostname }} {{MASTER}}:10000 + peer slave_lb {{SLAVE}}:10000 + {% else %} + peer master_lb {{MASTER}}:10000 + peer {{ ansible_hostname }} {{SLAVE}}:10000 + {% endif %} +{% endif %} + backend per_ip_and_url_rates - stick-table type binary len 8 size 1m expire 24h store http_req_rate(24h) + stick-table type binary len 8 size 1m expire 24h store http_req_rate(24h) {% if M_OR_S != 'None' %} peers default_peers {% endif %} + backend per_ip_rates - stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s) + stick-table type ip size 1m expire 24h store gpc0,gpc0_rate(30s) {% if M_OR_S != 'None' %} peers default_peers {% endif %} diff --git a/app/scripts/install_haproxy.sh b/app/scripts/install_haproxy.sh index c9581b50..688923bd 100644 --- a/app/scripts/install_haproxy.sh +++ b/app/scripts/install_haproxy.sh @@ -5,22 +5,25 @@ do VALUE=$(echo $ARGUMENT | cut -f2 -d=) case "$KEY" in - PROXY) PROXY=${VALUE} ;; + PROXY) PROXY=${VALUE} ;; SOCK_PORT) SOCK_PORT=${VALUE} ;; STAT_PORT) STAT_PORT=${VALUE} ;; STAT_FILE) STAT_FILE=${VALUE} ;; - STATS_USER) STATS_USER=${VALUE} ;; - STATS_PASS) STATS_PASS=${VALUE} ;; - HAPVER) HAPVER=${VALUE} ;; - HOST) HOST=${VALUE} ;; - USER) USER=${VALUE} ;; - PASS) PASS=${VALUE} ;; - KEY) KEY=${VALUE} ;; + STATS_USER) STATS_USER=${VALUE} ;; + STATS_PASS) STATS_PASS=${VALUE} ;; + HAPVER) HAPVER=${VALUE} ;; + HOST) HOST=${VALUE} ;; + USER) USER=${VALUE} ;; + PASS) PASS=${VALUE} ;; + KEY) KEY=${VALUE} ;; SYN_FLOOD) SYN_FLOOD=${VALUE} ;; - SSH_PORT) SSH_PORT=${VALUE} ;; - DOCKER) DOCKER=${VALUE} ;; - HAP_DIR) HAP_DIR=${VALUE} ;; + SSH_PORT) SSH_PORT=${VALUE} ;; + DOCKER) DOCKER=${VALUE} ;; + HAP_DIR) HAP_DIR=${VALUE} ;; CONT_NAME) CONT_NAME=${VALUE} ;; + M_OR_S) M_OR_S=${VALUE} ;; + MASTER) MASTER=${VALUE} ;; + SLAVE) SLAVE=${VALUE} ;; *) esac done @@ -41,9 +44,9 @@ else fi if [[ $KEY == "" ]]; then - ansible-playbook $PWD/roles/haproxy.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY HAPVER=$HAPVER HAP_DIR=$HAP_DIR CONT_NAME=$CONT_NAME SOCK_PORT=$SOCK_PORT STAT_PORT=$STAT_PORT STATS_USER=$STATS_USER STATS_PASS='$STATS_PASS' STAT_FILE=$STAT_FILE SSH_PORT=$SSH_PORT SYN_FLOOD=$SYN_FLOOD" -i $PWD/$HOST -t $tags + ansible-playbook $PWD/roles/haproxy.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY HAPVER=$HAPVER HAP_DIR=$HAP_DIR CONT_NAME=$CONT_NAME SOCK_PORT=$SOCK_PORT STAT_PORT=$STAT_PORT STATS_USER=$STATS_USER STATS_PASS='$STATS_PASS' STAT_FILE=$STAT_FILE SSH_PORT=$SSH_PORT SYN_FLOOD=$SYN_FLOOD M_OR_S=$M_OR_S MASTER=$MASTER SLAVE=$SLAVE" -i $PWD/$HOST -t $tags else - ansible-playbook $PWD/roles/haproxy.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY HAPVER=$HAPVER HAP_DIR=$HAP_DIR CONT_NAME=$CONT_NAME SOCK_PORT=$SOCK_PORT STAT_PORT=$STAT_PORT STATS_USER=$STATS_USER STATS_PASS='$STATS_PASS' STAT_FILE=$STAT_FILE SSH_PORT=$SSH_PORT SYN_FLOOD=$SYN_FLOOD" -i $PWD/$HOST -t $tags + ansible-playbook $PWD/roles/haproxy.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY HAPVER=$HAPVER HAP_DIR=$HAP_DIR CONT_NAME=$CONT_NAME SOCK_PORT=$SOCK_PORT STAT_PORT=$STAT_PORT STATS_USER=$STATS_USER STATS_PASS='$STATS_PASS' STAT_FILE=$STAT_FILE SSH_PORT=$SSH_PORT SYN_FLOOD=$SYN_FLOOD M_OR_S=$M_OR_S MASTER=$MASTER SLAVE=$SLAVE" -i $PWD/$HOST -t $tags fi if [ $? -gt 0 ] diff --git a/app/templates/add.html b/app/templates/add.html index 4d70be6e..07cc495f 100644 --- a/app/templates/add.html +++ b/app/templates/add.html @@ -33,6 +33,7 @@
  • {{lang.words.userlists|title()}}
  • Peers
  • {{lang.words.lists|title()}}
    • +
  • {{lang.words.maps|title()}}
    • {% include 'include/add_proxy.html' %} @@ -1084,7 +1085,7 @@ {% for list in black_lists %} - {{ list }} + {{ list }} {% endfor %} @@ -1126,11 +1127,53 @@ {{lang.add_page.desc.lists_howto}} {{lang.words.article}} - + +
    + + + + + + + + + + + + {{ input('group', value=group, type='hidden') }} +
    {{lang.words.new|title()}} {{lang.words.map}}{{lang.words.server|title()}} {{lang.words.for}} {{lang.words.uploading}}{{lang.words.existing|title()}} {{lang.words.maps}}
    + {{lang.words.name|title()}}: {{ input('new_map_name') }} + + + + + {% for map in maps %} + + {{ map }} + + {% endfor %} +
    +
    +
    + {{lang.add_page.desc.lists_howto}} + {{lang.words.article}} +
    + +
    {% include 'include/del_confirm.html' %}