From c6e6988209a6a46c17ae508f587828bfb2383f1b Mon Sep 17 00:00:00 2001 From: Aidaho Date: Fri, 30 Aug 2024 17:25:31 +0300 Subject: [PATCH] v8.0: Update credential endpoints and proxy settings Replaced hostname lookups with unified server object retrieval in metrics and credential management endpoints. Added proxy environment variables to ModSecurity download tasks and improved error handling across various modules. --- app/modules/db/server.py | 9 ------ app/modules/roxywi/common.py | 8 +++-- app/modules/service/installation.py | 7 ++++- app/modules/tools/alerting.py | 15 ++++++--- app/routes/metric/routes.py | 10 +++--- app/routes/server/routes.py | 2 +- app/scripts/ansible/roles/waf/tasks/main.yml | 12 +++++++ app/views/server/cred_views.py | 33 +++++++++----------- 8 files changed, 54 insertions(+), 42 deletions(-) diff --git a/app/modules/db/server.py b/app/modules/db/server.py index f600986f..0fafefa2 100644 --- a/app/modules/db/server.py +++ b/app/modules/db/server.py @@ -36,15 +36,6 @@ def update_server(hostname, ip, group, type_ip, enable, master, server_id, cred, out_error(e) -def get_hostname_by_server_ip(server_ip): - try: - hostname = Server.get(Server.ip == server_ip) - except Exception as e: - return out_error(e) - else: - return hostname.hostname - - def get_server_by_id(server_id: int) -> Server: try: return Server.get(Server.server_id == server_id) diff --git a/app/modules/roxywi/common.py b/app/modules/roxywi/common.py index 1853fa45..d1989b4f 100644 --- a/app/modules/roxywi/common.py +++ b/app/modules/roxywi/common.py @@ -166,6 +166,7 @@ def keep_action_history(service: str, action: str, server_ip: str, login: str, u hostname = ha_sql.select_cluster_name(int(server_id)) except Exception as e: logging('Roxy-WI server', f'Cannot get info about cluster {server_ip} for history: {e}', roxywi=1) + return elif service == 'UDP listener': try: server_id = int(server_ip) @@ -173,12 +174,15 @@ def keep_action_history(service: str, action: str, server_ip: str, login: str, u hostname = listener.name except Exception as e: logging('Roxy-WI server', f'Cannot get info about Listener {server_ip} for history: {e}', roxywi=1) + return else: try: - server_id = server_sql.select_server_id_by_ip(server_ip=server_ip) - hostname = server_sql.get_hostname_by_server_ip(server_ip) + server = server_sql.get_server_by_ip(server_ip) + server_id = server.server_id + hostname = server.hostname except Exception as e: logging('Roxy-WI server', f'Cannot get info about {server_ip} for history: {e}', roxywi=1) + return try: history_sql.insert_action_history(service, action, server_id, user_id, user_ip, server_ip, hostname) diff --git a/app/modules/service/installation.py b/app/modules/service/installation.py index 809641cb..261b9284 100644 --- a/app/modules/service/installation.py +++ b/app/modules/service/installation.py @@ -348,12 +348,17 @@ def _install_ansible_collections(): old_ansible_server = '' collections = ('community.general', 'ansible.posix', 'community.docker', 'community.grafana', 'ansible.netcommon') trouble_link = 'Read troubleshooting' + proxy = sql.get_setting('proxy') + proxy_cmd = '' + if proxy is not None and proxy != '' and proxy != 'None': + proxy_cmd = f'HTTPS_PROXY={proxy} &&' + for collection in collections: if not os.path.isdir(f'/usr/share/httpd/.ansible/collections/ansible_collections/{collection.replace(".", "/")}'): try: if version.parse(ansible.__version__) < version.parse('2.13.9'): old_ansible_server = '--server https://old-galaxy.ansible.com/' - exit_code = os.system(f'ansible-galaxy collection install {collection} {old_ansible_server}') + exit_code = os.system(f'{proxy_cmd} ansible-galaxy collection install {collection} {old_ansible_server}') except Exception as e: roxywi_common.handle_exceptions(e, 'Roxy-WI server', diff --git a/app/modules/tools/alerting.py b/app/modules/tools/alerting.py index 9753a6c3..4c32f45d 100644 --- a/app/modules/tools/alerting.py +++ b/app/modules/tools/alerting.py @@ -52,9 +52,12 @@ def send_message_to_rabbit(message: str, **kwargs) -> None: def alert_routing( server_ip: str, service_id: int, group_id: int, level: str, mes: str, alert_type: str ) -> None: - subject: str = level + ': ' + mes - server_id: int = server_sql.select_server_id_by_ip(server_ip) - checker_settings = checker_sql.select_checker_settings_for_server(service_id, server_id) + try: + subject: str = level + ': ' + mes + server_id: int = server_sql.select_server_id_by_ip(server_ip) + checker_settings = checker_sql.select_checker_settings_for_server(service_id, server_id) + except Exception as e: + raise Exception(f'Cannot get settings: {e}') try: json_for_sending = {"user_group": group_id, "message": subject} @@ -254,11 +257,13 @@ def pd_send_mess(mess, level, server_ip=None, service_id=None, alert_type=None, try: proxy = sql.get_setting('proxy') session = pdpyras.EventsAPISession(token) - dedup_key = f'{server_ip} {service_id} {alert_type}' + if server_ip: + dedup_key = f'{server_ip} {service_id} {alert_type}' + else: + dedup_key = f'{level}: {mess}' except Exception as e: roxywi_common.logging('Roxy-WI server', str(e), roxywi=1) raise Exception(e) - if proxy is not None and proxy != '' and proxy != 'None': proxies = dict(https=proxy, http=proxy) session.proxies.update(proxies) diff --git a/app/routes/metric/routes.py b/app/routes/metric/routes.py index 9e40d872..a2210935 100644 --- a/app/routes/metric/routes.py +++ b/app/routes/metric/routes.py @@ -101,13 +101,13 @@ def table_metrics(service): @bp.post('//') def show_metric(service, server_ip): server_ip = common.is_ip_or_dns(server_ip) - hostname = server_sql.get_hostname_by_server_ip(server_ip) + server = server_sql.get_server_by_ip(server_ip) time_range = common.checkAjaxInput(request.form.get('time_range')) if service in ('nginx', 'apache', 'waf'): - return jsonify(metric.service_metrics(server_ip, hostname, service, time_range)) + return jsonify(metric.service_metrics(server_ip, server.hostname, service, time_range)) elif service == 'haproxy': - return jsonify(metric.haproxy_metrics(server_ip, hostname, time_range)) + return jsonify(metric.haproxy_metrics(server_ip, server.hostname, time_range)) return 'error: Wrong service' @@ -116,11 +116,11 @@ def show_metric(service, server_ip): @check_services def show_http_metric(service, server_ip): server_ip = common.is_ip_or_dns(server_ip) - hostname = server_sql.get_hostname_by_server_ip(server_ip) + server = server_sql.get_server_by_ip(server_ip) time_range = common.checkAjaxInput(request.form.get('time_range')) if service == 'haproxy': - return jsonify(metric.haproxy_http_metrics(server_ip, hostname, time_range)) + return jsonify(metric.haproxy_http_metrics(server_ip, server.hostname, time_range)) return 'error: Wrong service' diff --git a/app/routes/server/routes.py b/app/routes/server/routes.py index 5d871059..2a7e3ac9 100644 --- a/app/routes/server/routes.py +++ b/app/routes/server/routes.py @@ -26,7 +26,7 @@ def register_api(view, endpoint, url, pk='listener_id', pk_type='int'): register_api(ServerView, 'server', '', 'server_id') register_api(ServerGroupView, 'group', '/group', 'group_id') -register_api(CredView, 'cred', '/cred', 'creds_id') +register_api(CredView, 'cred', '/cred', 'cred_id') bp.add_url_rule('/groups', view_func=ServerGroupsView.as_view('groups'), methods=['GET']) bp.add_url_rule('/creds', view_func=CredsView.as_view('creds'), methods=['GET']) diff --git a/app/scripts/ansible/roles/waf/tasks/main.yml b/app/scripts/ansible/roles/waf/tasks/main.yml index 315441eb..8f8533b4 100644 --- a/app/scripts/ansible/roles/waf/tasks/main.yml +++ b/app/scripts/ansible/roles/waf/tasks/main.yml @@ -111,6 +111,9 @@ url: "https://github.com/SpiderLabs/ModSecurity/releases/download/v{{ modsec_ver }}/modsecurity-{{ modsec_ver }}.tar.gz" dest: /tmp/modsecurity.tar.gz owner: "{{ ansible_user }}" + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" - name: Create HAProxy directory become: false @@ -192,6 +195,9 @@ get_url: url: https://github.com/SpiderLabs/ModSecurity/raw/v2/master/modsecurity.conf-recommended dest: "{{ SERVICE_PATH }}/waf/modsecurity.conf" + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" - name: Insert Modsec rules blockinfile: @@ -237,11 +243,17 @@ get_url: url: https://github.com/SpiderLabs/ModSecurity/raw/v2/master/unicode.mapping dest: "{{ SERVICE_PATH }}/waf/unicode.mapping" + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" - name: Download owasp-modsecurity-crs get_url: url: https://github.com/SpiderLabs/owasp-modsecurity-crs/archive/2.2.9.tar.gz dest: /tmp/owasp.tar.gz + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" - name: Create owasp directory file: diff --git a/app/views/server/cred_views.py b/app/views/server/cred_views.py index ccc934e8..51db76c4 100644 --- a/app/views/server/cred_views.py +++ b/app/views/server/cred_views.py @@ -82,11 +82,6 @@ class CredView(MethodView): tags: - SSH credentials parameters: - - in: 'path' - name: 'creds_id' - description: 'ID of the credential to retrieve' - required: true - type: 'integer' - in: body name: body schema: @@ -124,7 +119,7 @@ class CredView(MethodView): return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot create new cred') @validate(body=CredRequest) - def put(self, creds_id: int, body: CredRequest): + def put(self, cred_id: int, body: CredRequest): """ Update a credential entry --- @@ -132,7 +127,7 @@ class CredView(MethodView): - SSH credentials parameters: - in: 'path' - name: 'creds_id' + name: 'cred_id' description: 'ID of the credential to retrieve' required: true type: 'integer' @@ -167,17 +162,17 @@ class CredView(MethodView): """ group_id = SupportClass.return_group_id(body) try: - self._check_is_correct_group(creds_id) + self._check_is_correct_group(cred_id) except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, ''), 404 try: - ssh_mod.update_ssh_key(creds_id, body.name, body.password, body.key_enabled, body.username, group_id) + ssh_mod.update_ssh_key(cred_id, body.name, body.password, body.key_enabled, body.username, group_id) return BaseResponse().model_dump(mode='json'), 201 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot update SSH key') - def delete(self, creds_id: int): + def delete(self, cred_id: int): """ Delete a credential entry --- @@ -185,7 +180,7 @@ class CredView(MethodView): - SSH credentials parameters: - in: 'path' - name: 'creds_id' + name: 'cred_id' description: 'ID of the credential to retrieve' required: true type: 'integer' @@ -194,18 +189,18 @@ class CredView(MethodView): description: Credential deletion successful """ try: - self._check_is_correct_group(creds_id) + self._check_is_correct_group(cred_id) except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, ''), 404 try: - ssh_mod.delete_ssh_key(creds_id) + ssh_mod.delete_ssh_key(cred_id) return BaseResponse().model_dump(mode='json'), 204 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot delete SSH key') @validate(body=CredUploadRequest) - def patch(self, creds_id: int, body: CredUploadRequest): + def patch(self, cred_id: int, body: CredUploadRequest): """ Upload an SSH private key --- @@ -213,7 +208,7 @@ class CredView(MethodView): - SSH credentials parameters: - in: 'path' - name: 'creds_id' + name: 'cred_id' description: 'ID of the credential to retrieve' required: true type: 'integer' @@ -236,7 +231,7 @@ class CredView(MethodView): description: SSH key upload successful """ try: - self._check_is_correct_group(creds_id) + self._check_is_correct_group(cred_id) except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, ''), 404 try: @@ -244,17 +239,17 @@ class CredView(MethodView): except Exception: pass try: - ssh_mod.upload_ssh_key(creds_id, body.private_key, body.passphrase) + ssh_mod.upload_ssh_key(cred_id, body.private_key, body.passphrase) return BaseResponse().model_dump(mode='json'), 201 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot upload SSH key') @staticmethod - def _check_is_correct_group(creds_id: int): + def _check_is_correct_group(cred_id: int): if g.user_params['role'] == 1: return True try: - ssh = cred_sql.get_ssh(creds_id) + ssh = cred_sql.get_ssh(cred_id) except RoxywiResourceNotFound: raise RoxywiResourceNotFound if ssh.group_id != g.user_params['group_id']: