From c437274cfe080789b664b07cc5f13451f49d7934 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Sun, 3 Nov 2024 13:00:43 +0300 Subject: [PATCH] v8.1.2: Switch to retrieving server data by IP and hostname Updated multiple functions across the codebase to use the `get_server_by_ip` method instead of iterating over server lists. This change simplifies the code, improves readability, and reduces potential errors. Adjusted SQL queries to remove unnecessary conditions and parameters related to master servers. --- app/modules/db/server.py | 52 +++------------- app/modules/roxywi/common.py | 13 ++-- app/modules/roxywi/overview.py | 28 ++++----- app/modules/roxywi/waf.py | 78 ++++++++++++------------ app/modules/server/ssh.py | 27 ++++---- app/routes/admin/routes.py | 6 +- app/templates/include/admin_servers.html | 2 +- 7 files changed, 84 insertions(+), 122 deletions(-) diff --git a/app/modules/db/server.py b/app/modules/db/server.py index a2b036b2..05b36f4f 100644 --- a/app/modules/db/server.py +++ b/app/modules/db/server.py @@ -1,4 +1,4 @@ -from peewee import IntegrityError +from peewee import IntegrityError, DoesNotExist from app.modules.db.db_model import mysql_enable, connect, Server, SystemInfo from app.modules.db.common import out_error, not_unique_error @@ -38,7 +38,7 @@ def update_server(hostname, ip, group, type_ip, enable, master, server_id, cred, def get_server_by_id(server_id: int) -> Server: try: return Server.get(Server.server_id == server_id) - except Server.DoesNotExist: + except DoesNotExist: raise RoxywiResourceNotFound except Exception as e: return out_error(e) @@ -47,7 +47,7 @@ def get_server_by_id(server_id: int) -> Server: def get_server_by_ip(server_ip: str) -> Server: try: return Server.get(Server.ip == server_ip) - except Server.DoesNotExist: + except DoesNotExist: raise RoxywiResourceNotFound except Exception as e: return out_error(e) @@ -93,7 +93,7 @@ def is_system_info(server_id): def select_os_info(server_id): try: return SystemInfo.get(SystemInfo.server_id == server_id).os_info - except SystemInfo.DoesNotExist: + except DoesNotExist: raise RoxywiResourceNotFound except Exception as e: out_error(e) @@ -159,51 +159,15 @@ def select_servers(**kwargs): cursor = conn.cursor() if mysql_enable == '1': - sql = """select * from `servers` where `enabled` = 1 ORDER BY servers.group_id """ + sql = """select * from `servers` ORDER BY hostname """ if kwargs.get("server") is not None: sql = """select * from `servers` where `ip` = '{}' """.format(kwargs.get("server")) - if kwargs.get("full") is not None: - sql = """select * from `servers` ORDER BY hostname """ - if kwargs.get("get_master_servers") is not None: - sql = """select id,hostname from `servers` where `master` = 0 and type_ip = 0 and enabled = 1 ORDER BY servers.group_id """ - if kwargs.get("get_master_servers") is not None and kwargs.get('user_id') is not None: - sql = """ select servers.id, servers.hostname from `servers` - left join user as user on servers.group_id = user.group_id - where user.user_id = '{}' and servers.master = 0 and servers.type_ip = 0 and servers.enabled = 1 ORDER BY servers.group_id - """.format(kwargs.get('user_id')) - if kwargs.get("id"): - sql = """select * from `servers` where `id` = '{}' """.format(kwargs.get("id")) - if kwargs.get("hostname"): - sql = """select * from `servers` where `hostname` = '{}' """.format(kwargs.get("hostname")) - if kwargs.get("id_hostname"): - sql = """select * from `servers` where `hostname` ='{}' or id = '{}' or ip = '{}'""".format( - kwargs.get("id_hostname"), kwargs.get("id_hostname"), kwargs.get("id_hostname")) - if kwargs.get("server") and kwargs.get("keep_alive"): - sql = """select haproxy_active from `servers` where `ip` = '{}' """.format(kwargs.get("server")) else: - sql = """select * from servers where enabled = '1' ORDER BY servers.group_id """ + sql = """select * from servers ORDER BY hostname """ if kwargs.get("server") is not None: sql = """select * from servers where ip = '{}' """.format(kwargs.get("server")) - if kwargs.get("full") is not None: - sql = """select * from servers ORDER BY hostname """ - if kwargs.get("get_master_servers") is not None: - sql = """select id,hostname from servers where master = 0 and type_ip = 0 and enabled = 1 ORDER BY servers.group_id """ - if kwargs.get("get_master_servers") is not None and kwargs.get('user_id') is not None: - sql = """ select servers.id, servers.hostname from servers - left join user as user on servers.group_id = user.group_id - where user.user_id = '{}' and servers.master = 0 and servers.type_ip = 0 and servers.enabled = 1 ORDER BY servers.group_id - """.format(kwargs.get('user_id')) - if kwargs.get("id"): - sql = """select * from servers where id = '{}' """.format(kwargs.get("id")) - if kwargs.get("hostname"): - sql = """select * from servers where hostname = '{}' """.format(kwargs.get("hostname")) - if kwargs.get("id_hostname"): - sql = """select * from servers where hostname = '{}' or id = '{}' or ip = '{}'""".format( - kwargs.get("id_hostname"), kwargs.get("id_hostname"), kwargs.get("id_hostname")) - if kwargs.get("server") and kwargs.get("keep_alive"): - sql = """select haproxy_active from servers where ip = '{}' """.format(kwargs.get("server")) try: cursor.execute(sql) @@ -286,7 +250,7 @@ def is_master(ip, **kwargs): def get_server_with_group(server_id: int, group_id: int) -> Server: try: return Server.get((Server.server_id == server_id) & (Server.group_id == group_id)) - except Server.DoesNotExist: + except DoesNotExist: raise RoxywiResourceNotFound except Exception as e: out_error(e) @@ -295,7 +259,7 @@ def get_server_with_group(server_id: int, group_id: int) -> Server: def select_servers_with_group(group_id: int) -> Server: try: return Server.select().where(Server.group_id == group_id) - except Server.DoesNotExist: + except DoesNotExist: raise RoxywiResourceNotFound except Exception as e: out_error(e) diff --git a/app/modules/roxywi/common.py b/app/modules/roxywi/common.py index 933bb0cd..4892d9de 100644 --- a/app/modules/roxywi/common.py +++ b/app/modules/roxywi/common.py @@ -70,13 +70,12 @@ def check_user_group_for_socket(user_id: int, group_id: int) -> bool: def check_is_server_in_group(server_ip: str) -> bool: group_id = get_user_group(id=1) - servers = server_sql.select_servers(server=server_ip) - for s in servers: - if (s[2] == server_ip and int(s[3]) == int(group_id)) or group_id == 1: - return True - else: - logging('Roxy-WI server', 'has tried to actions in not his group server', roxywi=1, login=1) - return False + server = server_sql.get_server_by_ip(server_ip) + if (server.ip == server_ip and int(server.group_id) == int(group_id)) or group_id == 1: + return True + else: + logging('Roxy-WI server', 'has tried to actions in not his group server', roxywi=1, login=1) + return False def get_files(folder, file_format, server_ip=None) -> list: diff --git a/app/modules/roxywi/overview.py b/app/modules/roxywi/overview.py index c3ee7e6b..1889fb52 100644 --- a/app/modules/roxywi/overview.py +++ b/app/modules/roxywi/overview.py @@ -46,7 +46,7 @@ def show_overview(serv) -> str: claims = get_jwt() lang = roxywi_common.get_user_lang_for_flask() role = user_sql.get_user_role_in_group(claims['user_id'], claims['group']) - server = [server for server in server_sql.select_servers(server=serv)] + server = server_sql.get_server_by_ip(serv) user_services = user_sql.select_user_services(claims['user_id']) haproxy = service_sql.select_haproxy(serv) if '1' in user_services else 0 @@ -54,7 +54,7 @@ def show_overview(serv) -> str: keepalived = service_sql.select_keepalived(serv) if '3' in user_services else 0 apache = service_sql.select_apache(serv) if '4' in user_services else 0 - waf = waf_sql.select_waf_servers(server[0][2]) + waf = waf_sql.select_waf_servers(server.ip) haproxy_process = '' keepalived_process = '' nginx_process = '' @@ -67,43 +67,43 @@ def show_overview(serv) -> str: waf_len = 0 if haproxy: - cmd = f'echo "show info" |nc {server[0][2]} {sql.get_setting("haproxy_sock_port")} -w 1|grep -e "Process_num"' + cmd = f'echo "show info" |nc {server.ip} {sql.get_setting("haproxy_sock_port")} -w 1|grep -e "Process_num"' try: haproxy_process = service_common.server_status(server_mod.subprocess_execute(cmd)) except Exception as e: - return f'error: {e} for server {server[0][2]}' + return f'error: {e} for server {server.hostname}' if nginx: - nginx_cmd = f'echo "something" |nc {server[0][2]} {sql.get_setting("nginx_stats_port")} -w 1' + nginx_cmd = f'echo "something" |nc {server.ip} {sql.get_setting("nginx_stats_port")} -w 1' try: nginx_process = service_common.server_status(server_mod.subprocess_execute(nginx_cmd)) except Exception as e: - return f'error: {e} for server {server[0][2]}' + return f'error: {e} for server {server.hostname}' if apache: - apache_cmd = f'echo "something" |nc {server[0][2]} {sql.get_setting("apache_stats_port")} -w 1' + apache_cmd = f'echo "something" |nc {server.ip} {sql.get_setting("apache_stats_port")} -w 1' try: apache_process = service_common.server_status(server_mod.subprocess_execute(apache_cmd)) except Exception as e: - return f'error: {e} for server {server[0][2]}' + return f'error: {e} for server {server.hostname}' if keepalived: command = "ps ax |grep keepalived|grep -v grep|wc -l|tr -d '\n'" try: - keepalived_process = server_mod.ssh_command(server[0][2], command) + keepalived_process = server_mod.ssh_command(server.ip, command) except Exception as e: - return f'error: {e} for server {server[0][2]}' + return f'error: {e} for server {server.hostname}' if waf_len >= 1: command = "ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l" try: - waf_process = server_mod.ssh_command(server[0][2], command) + waf_process = server_mod.ssh_command(server.ip, command) except Exception as e: - return f'error: {e} for server {server[0][2]}' + return f'error: {e} for server {server.hostname}' server_status = ( - server[0][1], server[0][2], haproxy, haproxy_process, waf_process, waf, keepalived, keepalived_process, nginx, - nginx_process, server[0][0], apache, apache_process + server.hostname, server.ip, haproxy, haproxy_process, waf_process, waf, keepalived, keepalived_process, nginx, + nginx_process, server.server_id, apache, apache_process ) servers.append(server_status) diff --git a/app/modules/roxywi/waf.py b/app/modules/roxywi/waf.py index e002e549..6b8eed84 100644 --- a/app/modules/roxywi/waf.py +++ b/app/modules/roxywi/waf.py @@ -11,58 +11,60 @@ import app.modules.roxywi.common as roxywi_common def waf_overview(serv: str, waf_service: str, claims: dict) -> str: - servers = server_sql.select_servers(server=serv) + # servers = server_sql.select_servers(server=serv) + server = server_sql.get_server_by_ip(serv) role = user_sql.get_user_role_in_group(claims['user_id'], claims['group']) returned_servers = [] waf = '' + waf_len = 0 metrics_en = 0 waf_process = '' waf_mode = '' is_waf_on_server = 0 - for server in servers: + # for server in servers: + if waf_service == 'haproxy': + is_waf_on_server = service_sql.select_haproxy(server.ip) + elif waf_service == 'nginx': + is_waf_on_server = service_sql.select_nginx(server.ip) + + if is_waf_on_server == 1: + config_path = sql.get_setting(f'{waf_service}_dir') if waf_service == 'haproxy': - is_waf_on_server = service_sql.select_haproxy(server[2]) + waf = waf_sql.select_waf_servers(server.ip) + metrics_en = waf_sql.select_waf_metrics_enable_server(server.ip) elif waf_service == 'nginx': - is_waf_on_server = service_sql.select_nginx(server[2]) + waf = waf_sql.select_waf_nginx_servers(server.ip) + try: + waf_len = len(waf) + except Exception: + waf_len = 0 - if is_waf_on_server == 1: - config_path = sql.get_setting(f'{waf_service}_dir') + if waf_len >= 1: if waf_service == 'haproxy': - waf = waf_sql.select_waf_servers(server[2]) - metrics_en = waf_sql.select_waf_metrics_enable_server(server[2]) + command = "ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l" elif waf_service == 'nginx': - waf = waf_sql.select_waf_nginx_servers(server[2]) - try: - waf_len = len(waf) - except Exception: - waf_len = 0 + command = f"grep 'modsecurity on' {common.return_nice_path(config_path)}* --exclude-dir=waf -Rs |wc -l" + commands1 = f"grep SecRuleEngine {config_path}/waf/modsecurity.conf |grep -v '#' |awk '{{print $2}}'" + waf_process = server_mod.ssh_command(server.ip, command) + waf_mode = server_mod.ssh_command(server.ip, commands1).strip() - if waf_len >= 1: - if waf_service == 'haproxy': - command = "ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l" - elif waf_service == 'nginx': - command = f"grep 'modsecurity on' {common.return_nice_path(config_path)}* --exclude-dir=waf -Rs |wc -l" - commands1 = f"grep SecRuleEngine {config_path}/waf/modsecurity.conf |grep -v '#' |awk '{{print $2}}'" - waf_process = server_mod.ssh_command(server[2], command) - waf_mode = server_mod.ssh_command(server[2], commands1).strip() - - server_status = (server[1], - server[2], - waf_process, - waf_mode, - metrics_en, - waf_len, - server[0]) - else: - server_status = (server[1], - server[2], - waf_process, - waf_mode, - metrics_en, - waf_len, - server[0]) - returned_servers.append(server_status) + server_status = (server.hostname, + server.ip, + waf_process, + waf_mode, + metrics_en, + waf_len, + server.server_id) + else: + server_status = (server.hostname, + server.ip, + waf_process, + waf_mode, + metrics_en, + waf_len, + server.server_id) + returned_servers.append(server_status) lang = roxywi_common.get_user_lang_for_flask() servers_sorted = sorted(returned_servers, key=common.get_key) diff --git a/app/modules/server/ssh.py b/app/modules/server/ssh.py index f90568de..f853bb35 100644 --- a/app/modules/server/ssh.py +++ b/app/modules/server/ssh.py @@ -99,21 +99,22 @@ def upload_ssh_key(ssh_id: int, key: str, passphrase: str) -> None: name = ssh.name ssh_keys = f'{full_dir}{name}_{group_name}.pem' - if key != '': - try: - key = paramiko.pkey.load_private_key(key, password=passphrase) - except Exception as e: - raise Exception(e) + if key == '': + raise ValueError('Private key cannot be empty') + try: + key = paramiko.pkey.load_private_key(key, password=passphrase) + except Exception as e: + raise e - try: - key.write_private_key_file(ssh_keys) - except Exception as e: - raise Exception(e) + try: + key.write_private_key_file(ssh_keys) + except Exception as e: + raise e - try: - os.chmod(ssh_keys, 0o600) - except IOError as e: - raise Exception(e) + try: + os.chmod(ssh_keys, 0o600) + except IOError as e: + raise Exception(e) if passphrase: try: diff --git a/app/routes/admin/routes.py b/app/routes/admin/routes.py index cdcb13d5..23e7afc4 100644 --- a/app/routes/admin/routes.py +++ b/app/routes/admin/routes.py @@ -38,13 +38,11 @@ def admin(): user_group = roxywi_common.get_user_group(id=1) if g.user_params['role'] == 1: users = user_sql.select_users() - servers = server_sql.select_servers(full=1) - masters = server_sql.select_servers(get_master_servers=1) + servers = server_sql.select_servers() sshs = ssh_mod.get_creds() else: users = user_sql.select_users(group=user_group) servers = roxywi_common.get_dick_permit(virt=1, disable=0, only_group=1) - masters = server_sql.select_servers(get_master_servers=1, uuid=g.user_params['user_id']) sshs = ssh_mod.get_creds(group_id=user_group) kwargs = { @@ -57,8 +55,6 @@ def admin(): 'roles': sql.select_roles(), 'ldap_enable': sql.get_setting('ldap_enable'), 'services': service_sql.select_services(), - 'masters': masters, - 'guide_me': 1, 'user_subscription': roxywi_common.return_user_subscription(), 'users_roles': user_sql.select_users_roles(), 'user_roles': user_sql.select_user_roles_by_group(user_group), diff --git a/app/templates/include/admin_servers.html b/app/templates/include/admin_servers.html index 6278ba19..e2bd979f 100644 --- a/app/templates/include/admin_servers.html +++ b/app/templates/include/admin_servers.html @@ -103,7 +103,7 @@