From e53a7445c7ce745ce79b0eb575e090ec988deea7 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Sat, 4 Jan 2025 10:49:28 +0300 Subject: [PATCH] v8.1.5: Add UDP backend status checks and refine cluster HA handling Introduced backend status monitoring for UDP listeners and enhanced HA cluster checks. Updated several templates and JavaScript files to reflect these changes, improving service visibility and coordination. Minor code refactoring and removed unused functions for cleaner implementation. --- app/api/routes/routes.py | 3 +- app/create_db.py | 2 +- app/modules/db/keep_alive.py | 23 +++------- app/modules/db/roxy.py | 7 --- app/modules/roxywi/roxy.py | 2 +- app/routes/ha/routes.py | 23 ++++++++++ app/routes/udp/routes.py | 3 +- app/static/js/channel.js | 8 ++++ app/static/js/ha.js | 50 +++++++++++++++++++++ app/static/js/overview.js | 3 ++ app/static/js/udp.js | 34 ++++++++++++++ app/templates/ajax/channels.html | 4 +- app/templates/ajax/ha/clusters.html | 3 +- app/templates/service.html | 3 +- app/templates/udp/listener.html | 11 ++++- app/views/udp/views.py | 70 ++++++++++++++++++++++++++++- 16 files changed, 213 insertions(+), 36 deletions(-) diff --git a/app/api/routes/routes.py b/app/api/routes/routes.py index f5326783..0c8aa3dd 100644 --- a/app/api/routes/routes.py +++ b/app/api/routes/routes.py @@ -16,7 +16,7 @@ from app.views.service.lets_encrypt_views import LetsEncryptsView, LetsEncryptVi from app.views.service.haproxy_lists_views import HaproxyListView from app.views.ha.views import HAView, HAVIPView, HAVIPsView from app.views.user.views import UserView, UserGroupView, UserRoles -from app.views.udp.views import UDPListener, UDPListeners, UDPListenerActionView +from app.views.udp.views import UDPListener, UDPListeners, UDPListenerActionView, UDPListenerBackendStatusView from app.views.channel.views import ChannelView, ChannelsView from app.views.tools.views import CheckerView from app.views.tools.port_scanner_views import PortScannerView, PortScannerPortsView @@ -59,6 +59,7 @@ register_api(HAVIPView, 'ha_vip', '/ha///vip', 'vip_id' bp.add_url_rule('/ha///vips', view_func=HAVIPsView.as_view('ha_vips'), methods=['GET']) register_api(UDPListener, 'udp_listener', '//listener', 'listener_id') bp.add_url_rule('//listener//', view_func=UDPListenerActionView.as_view('listener_action'), methods=['GET']) +bp.add_url_rule('//listener//', view_func=UDPListenerBackendStatusView.as_view('UDPListenerBackendStatusView'), methods=['GET']) bp.add_url_rule('//listeners', view_func=UDPListeners.as_view('listeners'), methods=['GET']) bp.add_url_rule('/service///install', view_func=InstallGetStatus.as_view('install_status'), methods=['GET']) bp.add_url_rule('/service///install', view_func=InstallGetStatus.as_view('install_status_ip'), methods=['GET']) diff --git a/app/create_db.py b/app/create_db.py index 75f502c9..30ed5d03 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -705,7 +705,7 @@ def update_db_v_8_1_4(): def update_ver(): try: - Version.update(version='8.1.4').execute() + Version.update(version='8.1.5').execute() except Exception: print('Cannot update version') diff --git a/app/modules/db/keep_alive.py b/app/modules/db/keep_alive.py index 2f517df1..48bb472a 100644 --- a/app/modules/db/keep_alive.py +++ b/app/modules/db/keep_alive.py @@ -3,43 +3,31 @@ from app.modules.db.common import out_error def select_keep_alive(): - query = Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.haproxy_active == 1) try: - query_res = query.execute() + return Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.haproxy_active == 1).execute() except Exception as e: out_error(e) - else: - return query_res def select_nginx_keep_alive(): - query = Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.nginx_active == 1) try: - query_res = query.execute() + return Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.nginx_active == 1).execute() except Exception as e: out_error(e) - else: - return query_res def select_apache_keep_alive(): - query = Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.apache_active == 1) try: - query_res = query.execute() + return Server.select(Server.ip, Server.group_id, Server.server_id).where(Server.apache_active == 1).execute() except Exception as e: out_error(e) - else: - return query_res def select_keepalived_keep_alive(): - query = Server.select(Server.ip, Server.port, Server.group_id, Server.server_id).where(Server.keepalived_active == 1) try: - query_res = query.execute() + return Server.select(Server.ip, Server.port, Server.group_id, Server.server_id).where(Server.keepalived_active == 1).execute() except Exception as e: out_error(e) - else: - return query_res def select_update_keep_alive_restart(server_id: int, service: str) -> int: @@ -55,8 +43,7 @@ def select_update_keep_alive_restart(server_id: int, service: str) -> int: def update_keep_alive_restart(server_id: int, service: str, restarted: int) -> None: - query = KeepaliveRestart.insert(server_id=server_id, service=service, restarted=restarted).on_conflict('replace') try: - query.execute() + KeepaliveRestart.insert(server_id=server_id, service=service, restarted=restarted).on_conflict('replace').execute() except Exception as e: out_error(e) diff --git a/app/modules/db/roxy.py b/app/modules/db/roxy.py index a4848eef..d7b59763 100644 --- a/app/modules/db/roxy.py +++ b/app/modules/db/roxy.py @@ -23,13 +23,6 @@ def get_user() -> UserName: print(str(e)) -def select_user_status() -> int: - try: - return UserName.get().Status - except Exception: - return 0 - - def get_roxy_tools(): try: query_res = RoxyTool.select().where(RoxyTool.is_roxy == 1).execute() diff --git a/app/modules/roxywi/roxy.py b/app/modules/roxywi/roxy.py index 7a7f97bb..80161ca6 100644 --- a/app/modules/roxywi/roxy.py +++ b/app/modules/roxywi/roxy.py @@ -103,7 +103,7 @@ def action_service(action: str, service: str) -> str: if not re.match(r'^[a-zA-Z0-9\.\-]+$', service): return f"Invalid service name: {service}. Only alphanumeric characters, dots, and hyphens are allowed." cmd = f"sudo systemctl {actions[action]} {service}" - if not roxy_sql.select_user_status(): + if not roxy_sql.get_user().Status: return 'warning: The service is disabled because you are not subscribed. Read here about subscriptions' if is_in_docker: diff --git a/app/routes/ha/routes.py b/app/routes/ha/routes.py index 6b92a097..77a1296a 100644 --- a/app/routes/ha/routes.py +++ b/app/routes/ha/routes.py @@ -150,3 +150,26 @@ def get_masters(service): return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot get free servers') return jsonify([model_to_dict(free) for free in free_servers]) + + +@bp.route('///status') +@check_services +@get_user_params() +def check_cluster_status(service: str, cluster_id: int): + try: + router_id = ha_sql.get_router_id(cluster_id, default_router=1) + slaves = ha_sql.select_cluster_slaves(cluster_id, router_id) + except Exception as e: + return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot get slaves') + status = 'ok' + statuses = [] + cmd = f'systemctl is-active keepalived.service' + for slave in slaves: + status = server_mod.ssh_command(slave[2], cmd) + statuses.append(status.replace('\n', '').replace('\r', '')) + if 'inactive' in statuses and 'active' in statuses: + status = 'warning' + elif 'inactive' in statuses and 'active' not in statuses: + status = 'error' + + return jsonify({'status': status}) \ No newline at end of file diff --git a/app/routes/udp/routes.py b/app/routes/udp/routes.py index 804c1725..ca61a4c8 100644 --- a/app/routes/udp/routes.py +++ b/app/routes/udp/routes.py @@ -1,7 +1,7 @@ from flask_jwt_extended import jwt_required from app.routes.udp import bp -from app.views.udp.views import UDPListener +from app.views.udp.views import UDPListener, UDPListenerBackendStatusView @bp.before_request @@ -13,3 +13,4 @@ def before_request(): bp.add_url_rule('//listener', view_func=UDPListener.as_view('udp_listener', False), methods=['GET'], defaults={'listener_id': None}) bp.add_url_rule('//listener/', view_func=UDPListener.as_view('udp_listener_id', False), methods=['GET']) +bp.add_url_rule('//listener//', view_func=UDPListenerBackendStatusView.as_view('udp_listener_backend_ip'), methods=['GET']) diff --git a/app/static/js/channel.js b/app/static/js/channel.js index a008be83..8fdf6024 100644 --- a/app/static/js/channel.js +++ b/app/static/js/channel.js @@ -151,6 +151,14 @@ $( function() { let id = $(this).attr('id').split('-'); updateReceiver(id[1], 'pd') }); + $("#checker_mm_table input").change(function () { + let id = $(this).attr('id').split('-'); + updateReceiver(id[2], 'mm') + }); + $("#checker_pd_tablechecker_mm_table select").on('selectmenuchange', function () { + let id = $(this).attr('id').split('-'); + updateReceiver(id[1], 'mm') + }); }); function loadChannel() { $.ajax({ diff --git a/app/static/js/ha.js b/app/static/js/ha.js index 51132062..e76236ad 100644 --- a/app/static/js/ha.js +++ b/app/static/js/ha.js @@ -864,3 +864,53 @@ function getHaCluster(cluster_id, new_cluster=false) { } }); } +function checkHaClusterStatus(cluster_id) { + if (sessionStorage.getItem('check-ha-cluster-'+cluster_id) == 0) { + return false; + } + NProgress.configure({showSpinner: false}); + let listener_div = $('#cluster-' + cluster_id); + $.ajax({ + url: "/ha/cluster/" + cluster_id + "/status", + contentType: "application/json; charset=utf-8", + statusCode: { + 404: function (xhr) { + $('#cluster-' + cluster_id).remove(); + }, + 403: function (xhr) { + sessionStorage.setItem('check-ha-cluster-'+cluster_id, 0); + }, + 500: function (xhr) { + sessionStorage.setItem('ccheck-ha-cluster-'+cluster_id, 0); + } + }, + success: function (data) { + try { + if (data.indexOf('logout') != '-1') { + sessionStorage.setItem('check-ha-cluster-'+cluster_id, 0); + } + } catch (e) {} + + if (data.status === 'ok') { + listener_div.addClass('div-server-head-up'); + listener_div.attr('title', 'All services are UP'); + listener_div.removeClass('div-server-head-down'); + listener_div.removeClass('div-server-head-unknown'); + listener_div.removeClass('div-server-head-dis'); + } else if (data.status === 'failed' || data.status === 'error') { + listener_div.removeClass('div-server-head-unknown'); + listener_div.removeClass('div-server-head-up'); + listener_div.removeClass('div-server-head-dis'); + listener_div.addClass('div-server-head-down'); + listener_div.attr('title', 'All services are DOWN'); + } else if (data.status === 'warning') { + listener_div.addClass('div-server-head-unknown'); + listener_div.removeClass('div-server-head-up'); + listener_div.removeClass('div-server-head-down'); + listener_div.removeClass('div-server-head-dis'); + listener_div.attr('title', 'Not all services are UP'); + } + } + }); + NProgress.configure({showSpinner: true}); +} diff --git a/app/static/js/overview.js b/app/static/js/overview.js index a5237d13..c07a1bfa 100644 --- a/app/static/js/overview.js +++ b/app/static/js/overview.js @@ -518,6 +518,7 @@ function check_service_status(id, ip, service) { } else { if (data.status === 'failed') { server_div.removeClass('div-server-head-unknown'); + server_div.removeClass('div-server-head-dis'); server_div.removeClass('div-server-head-up'); server_div.addClass('div-server-head-down'); } else { @@ -525,10 +526,12 @@ function check_service_status(id, ip, service) { server_div.addClass('div-server-head-up'); server_div.removeClass('div-server-head-down'); server_div.removeClass('div-server-head-unknown'); + server_div.removeClass('div-server-head-dis'); $('#uptime-word-'+id).text(translate_div.attr('data-uptime')); } else { server_div.removeClass('div-server-head-up'); server_div.removeClass('div-server-head-unknown'); + server_div.removeClass('div-server-head-dis'); server_div.addClass('div-server-head-down'); $('#uptime-word-'+id).text(translate_div.attr('data-downtime')); } diff --git a/app/static/js/udp.js b/app/static/js/udp.js index 3e62ab97..aebd5e01 100644 --- a/app/static/js/udp.js +++ b/app/static/js/udp.js @@ -549,8 +549,10 @@ function checkStatus(listener_id) { listener_div.attr('title', 'All services are UP'); listener_div.removeClass('div-server-head-down'); listener_div.removeClass('div-server-head-unknown'); + listener_div.removeClass('div-server-head-dis'); } else if (data.status === 'failed' || data.status === 'error') { listener_div.removeClass('div-server-head-unknown'); + listener_div.removeClass('div-server-head-dis'); listener_div.removeClass('div-server-head-up'); listener_div.addClass('div-server-head-down'); listener_div.attr('title', 'All services are DOWN'); @@ -558,6 +560,7 @@ function checkStatus(listener_id) { listener_div.addClass('div-server-head-unknown'); listener_div.removeClass('div-server-head-up'); listener_div.removeClass('div-server-head-down'); + listener_div.removeClass('div-server-head-dis'); listener_div.attr('title', 'Not all services are UP'); } $(`#listener-name-${listener_id}`).text(data.name.replaceAll("'", "")); @@ -572,3 +575,34 @@ function checkStatus(listener_id) { }); NProgress.configure({showSpinner: true}); } +function checkUdpBackendStatus(listener_id, backend_ip) { + $.ajax({ + url: `/udp/listener/${listener_id}/${backend_ip}`, + type: "GET", + contentType: "application/json; charset=utf-8", + success: function (data) { + if (data.status === 'failed') { + toastr.error(data.error); + return false; + } else { + let server_div = $('#backend_server_status_' + backend_ip.replaceAll('.', '')); + if (data.data === 'yes') { + server_div.removeClass('serverNone'); + server_div.removeClass('serverDown'); + server_div.addClass('serverUp'); + server_div.attr('title', 'Server is reachable'); + } else if (data.data === 'no') { + server_div.removeClass('serverNone'); + server_div.removeClass('serverUp'); + server_div.addClass('serverDown'); + server_div.attr('title', 'Server is unreachable'); + } else { + server_div.removeClass('serverDown'); + server_div.removeClass('serverUp'); + server_div.addClass('serverNone'); + server_div.attr('title', 'Cannot get server status'); + } + } + } + }); +} diff --git a/app/templates/ajax/channels.html b/app/templates/ajax/channels.html index 9bedd3ff..f8a40c35 100644 --- a/app/templates/ajax/channels.html +++ b/app/templates/ajax/channels.html @@ -160,9 +160,9 @@

Mattermost {{lang.words.channels|title()}}

- {{lang.words.key|title()}} + Webhook - Webhook + {{lang.words.channel|title()}} {{lang.words.name}} {% if user_params['role']|int() == 1 %} {{lang.words.group|title()}} {% endif %} diff --git a/app/templates/ajax/ha/clusters.html b/app/templates/ajax/ha/clusters.html index feee9449..a4ea5b5f 100644 --- a/app/templates/ajax/ha/clusters.html +++ b/app/templates/ajax/ha/clusters.html @@ -1,7 +1,7 @@ {% import 'languages/'+lang|default('en')+'.html' as lang %} {% from 'include/input_macros.html' import input, checkbox, copy_to_clipboard %} {% for cluster in clusters %} -
+ + {% endfor %} diff --git a/app/templates/service.html b/app/templates/service.html index 6dc461b9..065166ba 100644 --- a/app/templates/service.html +++ b/app/templates/service.html @@ -97,6 +97,7 @@ setInterval(showBytes, 60000, server_ip); {%- elif service == 'keepalived' %} keepalivedBecameMaster(server_ip) + setInterval(keepalivedBecameMaster, 60000, server_ip) {% endif %} } showMetrics(); @@ -181,7 +182,7 @@ {% set is_checker_enabled = s.4.0.8 %} {% set is_metrics_enabled = s.4.0.9 %} {% endif %} -
+
diff --git a/app/templates/udp/listener.html b/app/templates/udp/listener.html index e557d13d..86142283 100644 --- a/app/templates/udp/listener.html +++ b/app/templates/udp/listener.html @@ -1,6 +1,6 @@ {% import 'languages/'+lang|default('en')+'.html' as lang %} {% from 'include/input_macros.html' import input, checkbox, copy_to_clipboard %} -
+
{{listener.name|replace("'", "")}} @@ -38,7 +38,14 @@
{% set config = listener.config|string_to_dict %} {% for c in config %} - {{ lang.words.server|title() }}: {{ copy_to_clipboard(value=c.backend_ip) }}, {{ lang.words.port }}: {{ c.port }}, {{ lang.words.weight }}: {{ c.weight }}
+
+ + {{ lang.words.server|title() }}: {{ copy_to_clipboard(value=c.backend_ip) }}, {{ lang.words.port }}: {{ c.port }}, {{ lang.words.weight }}: {{ c.weight }} +
+ {% endfor %}
diff --git a/app/views/udp/views.py b/app/views/udp/views.py index a05af2aa..74a43749 100644 --- a/app/views/udp/views.py +++ b/app/views/udp/views.py @@ -1,8 +1,11 @@ +from typing import Union + from flask import render_template, g, jsonify from flask.views import MethodView from flask_pydantic import validate from flask_jwt_extended import jwt_required from playhouse.shortcuts import model_to_dict +from pydantic import IPvAnyAddress import app.modules.roxywi.auth as roxywi_auth import app.modules.roxywi.common as roxywi_common @@ -10,11 +13,12 @@ import app.modules.common.common as common import app.modules.db.udp as udp_sql import app.modules.db.ha_cluster as ha_sql import app.modules.db.server as server_sql +import app.modules.server.server as server_mod import app.modules.service.udp as udp_mod import app.modules.service.installation as service_mod from app.middleware import get_user_params, check_services, page_for_admin, check_group from app.modules.common.common_classes import SupportClass -from app.modules.roxywi.class_models import BaseResponse, IdResponse, UdpListenerRequest, GroupQuery +from app.modules.roxywi.class_models import BaseResponse, IdResponse, UdpListenerRequest, GroupQuery, DomainName, DataStrResponse class UDPListener(MethodView): @@ -498,3 +502,67 @@ class UDPListenerActionView(MethodView): return BaseResponse().model_dump(mode='json') except Exception as e: return roxywi_common.handle_json_exceptions(e, f'Cannot {action} listener') + + +class UDPListenerBackendStatusView(MethodView): + methods = ['GET'] + decorators = [jwt_required(), get_user_params(), check_services, page_for_admin(level=3), check_group()] + + @staticmethod + @validate() + def get(service: str, listener_id: int, backend_ip: Union[IPvAnyAddress, DomainName]): + """ + UDP Listener Backend Status View + + --- + tags: + - UDP listener + parameters: + - in: path + name: listener_id + required: true + description: The ID of the UDP listener. + type: integer + - in: path + name: backend_ip + required: true + description: The IP address of the backend server. + type: string + responses: + 200: + description: Success. Returns the backend status for the given UDP listener. + content: + application/json: + schema: + type: object + properties: + status: + type: string + description: The backend status (e.g., 'yes', 'no'). + 400: + description: Bad request. Invalid listener_id or backend_ip. + 404: + description: Not found. The specified UDP listener or backend was not found. + """ + try: + listener = udp_sql.get_listener(listener_id) + except Exception as e: + return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot get UDP listeners') + + if listener.cluster_id: + cluster = ha_sql.get_cluster(listener.cluster_id) + router_id = ha_sql.get_router_id(cluster.id, 1) + slaves = ha_sql.select_cluster_slaves(cluster.id, router_id) + + for slave in slaves: + server_ip = server_sql.get_server(slave[0]).ip + elif listener.server_id: + server_ip = server_sql.get_server(listener.server_id).ip + else: + return roxywi_common.handler_exceptions_for_json_data(Exception(''), 'Cannot get UDP listeners') + + cmd = (f"sudo kill -s $(keepalived --signum=DATA) $(cat /var/run/keepalived-udp-{listener_id}.pid) && " + f"sudo grep {backend_ip} -A 3 /tmp/keepalived_check.data |grep Up |awk '{{print $3}}'") + status = server_mod.ssh_command(server_ip, cmd) + status = status.replace('\r\n', '') + return DataStrResponse(data=status).model_dump(mode='json')