From 0391dff9ebadc35caa541ea8d8d0bf81e6208e19 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Tue, 27 Aug 2024 12:40:15 +0300 Subject: [PATCH] v8.0: Remove redundant requirement files and update VIP handling Removed outdated Debian, EL7, EL8, and EL9 requirement files. Improved various VIP handling and validation functionalities, including changes to VIP endpoints and associated JavaScript functions. Minor enhancements and bug fixes across multiple modules and views. --- app/modules/db/ha_cluster.py | 54 ++++-- app/modules/db/service.py | 21 +-- app/modules/roxywi/class_models.py | 17 +- app/modules/service/ha_cluster.py | 191 +++++++++++----------- app/routes/ha/routes.py | 20 ++- app/static/css/styles.css | 2 +- app/static/js/ha.js | 51 +++--- app/templates/ajax/ha/clusters.html | 3 +- app/templates/include/add_backup.html | 2 +- app/templates/include/admins_dialogs.html | 2 + app/views/ha/views.py | 141 +++++++--------- app/views/server/backup_vews.py | 46 +++++- app/views/service/views.py | 10 +- config_other/requirements_deb.txt | 23 --- config_other/requirements_el7.txt | 23 --- config_other/requirements_el8.txt | 24 --- config_other/requirements_el9.txt | 25 --- 17 files changed, 302 insertions(+), 353 deletions(-) delete mode 100644 config_other/requirements_deb.txt delete mode 100644 config_other/requirements_el7.txt delete mode 100644 config_other/requirements_el8.txt delete mode 100644 config_other/requirements_el9.txt diff --git a/app/modules/db/ha_cluster.py b/app/modules/db/ha_cluster.py index a4a6a6e..3fc2371 100644 --- a/app/modules/db/ha_cluster.py +++ b/app/modules/db/ha_cluster.py @@ -1,5 +1,6 @@ from app.modules.db.db_model import connect, HaCluster, HaClusterVirt, HaClusterVip, HaClusterService, HaClusterSlave, Server, HaClusterRouter from app.modules.db.common import out_error +from app.modules.roxywi.exception import RoxywiResourceNotFound def select_clusters(group_id: int): @@ -26,6 +27,13 @@ def select_cluster(cluster_id: int): out_error(e) +def get_cluster(cluster_id: int): + try: + return HaCluster.get(HaCluster.id == cluster_id) + except Exception as e: + out_error(e) + + def select_cluster_name(cluster_id: int) -> str: try: return HaCluster.get(HaCluster.id == cluster_id).name @@ -54,6 +62,13 @@ def select_cluster_vip(cluster_id: int, router_id: int) -> HaClusterVip: out_error(e) +def select_cluster_vip_by_vip_id(cluster_id: int, vip_id: int) -> HaClusterVip: + try: + return HaClusterVip.get((HaClusterVip.cluster_id == cluster_id) & (HaClusterVip.id == vip_id)) + except Exception as e: + out_error(e) + + def select_clusters_vip_id(cluster_id: int, router_id): try: return HaClusterVip.get((HaClusterVip.cluster_id == cluster_id) & (HaClusterVip.router_id == router_id)).id @@ -75,6 +90,15 @@ def insert_cluster_services(cluster_id: int, service_id: int): out_error(e) +def select_count_cluster_slaves(cluster_id: int) -> int: + try: + return HaClusterSlave.select().where(HaClusterSlave.cluster_id == cluster_id).count() + except HaClusterSlave.DoesNotExist: + raise RoxywiResourceNotFound + except Exception as e: + out_error(e) + + def select_cluster_master_slaves(cluster_id: int, group_id: int, router_id: int): conn = connect() cursor = conn.cursor() @@ -149,6 +173,8 @@ def get_router_id(cluster_id: int, default_router=0) -> int: """ try: return HaClusterRouter.get((HaClusterRouter.cluster_id == cluster_id) & (HaClusterRouter.default == default_router)).id + except HaClusterRouter.DoesNotExist: + raise RoxywiResourceNotFound except Exception as e: out_error(e) @@ -160,21 +186,22 @@ def get_router(router_id: int) -> HaClusterRouter: out_error(e) -def create_ha_router(cluster_id: int) -> int: +def create_ha_router(cluster_id: int, default: int = 0) -> int: """ - Create HA Router + Create HA Router - This method is used to create a HA (High Availability) router for a given cluster. + This method is used to create a HA (High Availability) router for a given cluster. - :param cluster_id: The ID of the cluster for which the HA router needs to be created. - :return: The ID of the created HA router. - :rtype: int + :param default: + :param cluster_id: The ID of the cluster for which the HA router needs to be created. + :return: The ID of the created HA router. + :rtype: int - :raises Exception: If an error occurs while creating the HA router. + :raises Exception: If an error occurs while creating the HA router. - """ + """ try: - last_id = HaClusterRouter.insert(cluster_id=cluster_id).execute() + last_id = HaClusterRouter.insert(cluster_id=cluster_id, default=default).on_conflict_ignore().execute() return last_id except Exception as e: out_error(e) @@ -256,15 +283,6 @@ def select_cluster_services(cluster_id: int): out_error(e) -def update_server_master(master, slave): - try: - master_id = Server.get(Server.ip == master).server_id - except Exception as e: - out_error(e) - - update_master_server_by_slave_ip(master_id, slave) - - def update_master_server_by_slave_ip(master_id: int, slave_ip: str) -> None: try: Server.update(master=master_id).where(Server.ip == slave_ip).execute() diff --git a/app/modules/db/service.py b/app/modules/db/service.py index 09c3faf..bda636e 100644 --- a/app/modules/db/service.py +++ b/app/modules/db/service.py @@ -150,24 +150,17 @@ def select_service_id_by_slug(service_slug: str) -> int: def select_services(): - query = Services.select() try: - query_res = query.execute() + return Services.select().execute() except Exception as e: out_error(e) - return - else: - return query_res -def select_service(slug: str) -> object: +def select_service(slug: str) -> Services: try: - query_res = Services.get(Services.slug == slug) + return Services.get(Services.slug == slug) except Exception as e: out_error(e) - return 'there is no service' - else: - return query_res def update_keepalived(serv): @@ -179,11 +172,9 @@ def update_keepalived(serv): def select_apache(serv): try: - apache = Server.get(Server.ip == serv).apache + return Server.get(Server.ip == serv).apache except Exception as e: out_error(e) - else: - return apache def update_apache(serv: str) -> None: @@ -195,11 +186,9 @@ def update_apache(serv: str) -> None: def select_nginx(serv): try: - query_res = Server.get(Server.ip == serv).nginx + return Server.get(Server.ip == serv).nginx except Exception as e: out_error(e) - else: - return query_res def update_nginx(serv: str) -> None: diff --git a/app/modules/roxywi/class_models.py b/app/modules/roxywi/class_models.py index 69e4c1c..c7da7b1 100644 --- a/app/modules/roxywi/class_models.py +++ b/app/modules/roxywi/class_models.py @@ -79,6 +79,9 @@ class UdpListenerRequest(BaseModel): lb_algo: Literal['rr', 'wrr', 'lc', 'wlc', 'sh', 'dh', 'wlc', 'lblc'] check_enabled: Optional[bool] = 1 reconfigure: Optional[bool] = 0 + delay_loop: Optional[int] = 10 + delay_before_retry: Optional[int] = 10 + retry: Optional[int] = 3 class UserPost(BaseModel): @@ -144,23 +147,23 @@ class CredUploadRequest(BaseModel): class HAClusterServer(BaseModel): eth: EscapedString id: int - ip: Union[IPvAnyAddress, DomainName] - name: EscapedString master: Optional[bool] = 1 +class HAClusterServersRequest(BaseModel): + servers: List[HAClusterServer] + + class HAClusterService(BaseModel): enabled: Optional[bool] = 0 docker: Optional[bool] = 0 class HAClusterVIP(BaseModel): - name: EscapedString use_src: Optional[bool] = 1 vip: IPvAnyAddress return_master: Optional[bool] = 1 virt_server: Optional[bool] = 1 - router_id: Optional[int] = None servers: List[HAClusterServer] @@ -168,16 +171,16 @@ class HAClusterRequest(BaseModel): name: EscapedString description: Optional[EscapedString] = None return_master: Optional[bool] = 1 - servers: List[HAClusterServer] + servers: Optional[List[HAClusterServer]] = None services: Dict[str, HAClusterService] syn_flood: Optional[bool] = 1 use_src: Optional[bool] = 1 - vip: IPvAnyAddress + vip: Optional[IPvAnyAddress] = None virt_server: Optional[bool] = 1 class ConfigFileNameQuery(BaseModel): - file_name: Optional[str] = None + file_path: Optional[str] = None version: Optional[str] = None diff --git a/app/modules/service/ha_cluster.py b/app/modules/service/ha_cluster.py index 6306687..00f7104 100644 --- a/app/modules/service/ha_cluster.py +++ b/app/modules/service/ha_cluster.py @@ -4,15 +4,16 @@ import app.modules.db.server as server_sql import app.modules.db.ha_cluster as ha_sql import app.modules.db.service as service_sql from app.modules.db.db_model import HaCluster, HaClusterRouter, HaClusterVip, HaClusterVirt -import app.modules.server.server as server_mod import app.modules.roxywi.common as roxywi_common -from app.modules.server.ssh import return_ssh_keys_path -from app.modules.roxywi.class_models import HAClusterRequest, HAClusterVIP +from app.modules.roxywi.class_models import HAClusterRequest, HAClusterVIP, HAClusterServersRequest +from app.modules.roxywi.exception import RoxywiResourceNotFound -def _get_servers_dict(cluster: Union[HAClusterRequest, HAClusterVIP]) -> dict: +def _get_servers_dict(cluster: Union[HAClusterRequest, HAClusterVIP, HAClusterServersRequest]) -> Union[dict, None]: for i, k in cluster.model_dump(mode='json').items(): if i == 'servers': + if k is None: + return None servers = k return servers @@ -25,7 +26,6 @@ def _get_services_dict(cluster: HAClusterRequest) -> dict: def create_cluster(cluster: HAClusterRequest, group_id: int) -> int: - master_ip = None servers = _get_servers_dict(cluster) services = _get_services_dict(cluster) @@ -35,39 +35,25 @@ def create_cluster(cluster: HAClusterRequest, group_id: int) -> int: except Exception as e: raise Exception(f'error: Cannot create new HA cluster: {e}') - try: - router_id = HaClusterRouter.insert(cluster_id=cluster_id, default=1).on_conflict_ignore().execute() - except Exception as e: - raise Exception(f'error: Cannon create router: {e}') - - try: - vip_id = HaClusterVip.insert(cluster_id=cluster_id, router_id=router_id, vip=cluster.vip, return_master=cluster.return_master).execute() - roxywi_common.logging(cluster_id, f'New vip {cluster.vip} has been created and added to the cluster', keep_history=1, roxywi=1, service='HA cluster') - except Exception as e: - raise Exception(f'error: Cannon add VIP: {e}') - - - for value in servers: - if value['master']: - master_ip = value['ip'] - - for value in servers: - if value['master']: - continue + if not servers is None: try: - ha_sql.update_server_master(master_ip, value['ip']) + router_id = ha_sql.create_ha_router(cluster_id, default=1) except Exception as e: - raise Exception(f'error: Cannot update master on slave {value["ip"]: {e}}') + raise Exception(f'error: Cannon create router: {e}') - for value in servers: - slave_id = value['id'] - if value['master']: - slave_id = server_sql.select_server_id_by_ip(master_ip) - try: - ha_sql.insert_or_update_slave(cluster_id, slave_id, value['eth'], value['master'], router_id) - roxywi_common.logging(cluster_id, f'New server {value["ip"]} has been added to the cluster', keep_history=1, roxywi=1, service='HA cluster') - except Exception as e: - raise Exception(f'error: Cannot update slave server {value["ip"]}: {e}') + _create_or_update_master_slaves_servers(cluster_id, servers, router_id, True) + + if cluster.vip: + try: + vip_id = HaClusterVip.insert(cluster_id=cluster_id, router_id=router_id, vip=cluster.vip, + return_master=cluster.return_master).execute() + roxywi_common.logging(cluster_id, f'New vip {cluster.vip} has been created and added to the cluster', + keep_history=1, roxywi=1, service='HA cluster') + except Exception as e: + raise Exception(f'error: Cannon add VIP: {e}') + + if cluster.virt_server and not servers is None: + add_or_update_virt(cluster, servers, cluster_id, vip_id, group_id) for service, value in services.items(): if not value['enabled']: @@ -79,9 +65,6 @@ def create_cluster(cluster: HAClusterRequest, group_id: int) -> int: except Exception as e: raise Exception(f'error: Cannot add service {service}: {e}') - if cluster.virt_server: - add_or_update_virt(cluster, servers, cluster_id, vip_id, group_id) - return int(cluster_id) @@ -89,25 +72,26 @@ def update_cluster(cluster: HAClusterRequest, cluster_id: int, group_id: int) -> servers = _get_servers_dict(cluster) services = _get_services_dict(cluster) - try: - router_id = ha_sql.get_router_id(cluster_id, default_router=1) - except Exception as e: - raise Exception(f'error: Cannot get router: {e}') - try: ha_sql.update_cluster(cluster_id, cluster.name, cluster.description, cluster.syn_flood) except Exception as e: raise Exception(f'error: Cannot update HA cluster: {e}') - try: - update_slaves(servers, cluster_id, router_id) - except Exception as e: - raise Exception(e) + if servers: + try: + router_id = ha_sql.get_router_id(cluster_id, default_router=1) + except Exception as e: + raise Exception(f'error: Cannot get router: {e}') - try: - update_vip(cluster_id, router_id, cluster, group_id) - except Exception as e: - raise Exception(e) + try: + update_slaves(servers, cluster_id, router_id) + except Exception as e: + raise Exception(e) + + try: + update_vip(cluster_id, router_id, cluster, group_id) + except Exception as e: + raise Exception(e) try: ha_sql.delete_cluster_services(cluster_id) @@ -126,7 +110,7 @@ def update_cluster(cluster: HAClusterRequest, cluster_id: int, group_id: int) -> roxywi_common.logging(cluster_id, f'Cluster {cluster.name} has been updated', keep_history=1, roxywi=1, service='HA cluster') -def delete_cluster(cluster_id: int) -> str: +def delete_cluster(cluster_id: int) -> None: router_id = ha_sql.get_router_id(cluster_id, default_router=1) slaves = ha_sql.select_cluster_slaves(cluster_id, router_id) @@ -137,11 +121,12 @@ def delete_cluster(cluster_id: int) -> str: except Exception as e: raise Exception(f'error: Cannot update master on slave {slave_ip}: {e}') - HaCluster.delete().where(HaCluster.id == cluster_id).execute() + try: + HaCluster.delete().where(HaCluster.id == cluster_id).execute() + except HaCluster.DoesNotExist: + raise RoxywiResourceNotFound roxywi_common.logging(cluster_id, 'Cluster has been deleted', roxywi=1, service='HA cluster') - return 'ok' - def update_vip(cluster_id: int, router_id: int, cluster: Union[HAClusterRequest, HAClusterVIP], group_id: int) -> None: vip_id = ha_sql.select_clusters_vip_id(cluster_id, router_id) @@ -156,8 +141,10 @@ def update_vip(cluster_id: int, router_id: int, cluster: Union[HAClusterRequest, try: ha_sql.update_slave(cluster_id, value['id'], value['eth'], value['master'], router_id) except Exception as e: - raise Exception(f'error: Cannot add server {value["ip"]}: {e}') + s = server_sql.get_server_by_id(value['id']) + raise Exception(f'error: Cannot add server {s.hostname}: {e}') + print('cluster.virt_server',cluster.virt_server) if cluster.virt_server: add_or_update_virt(cluster, servers, cluster_id, vip_id, group_id) else: @@ -171,14 +158,30 @@ def update_vip(cluster_id: int, router_id: int, cluster: Union[HAClusterRequest, roxywi_common.logging(cluster_id, f'Cluster VIP {cluster.vip} has been updated', keep_history=1, roxywi=1, service='HA cluster') -def insert_vip(cluster_id: int, cluster: HAClusterVIP, group_id: int) -> None: - vip = cluster.vip - servers = _get_servers_dict(cluster) +def insert_vip(cluster_id: int, cluster: HAClusterVIP, group_id: int) -> int: + try: + slaves_count = ha_sql.select_count_cluster_slaves(cluster_id) + if slaves_count == 0: + try: + router_id = ha_sql.create_ha_router(cluster_id, default=1) + except Exception as e: + raise Exception(f'error: Cannon create a new default router: {e}') + else: + try: + router_id = ha_sql.create_ha_router(cluster_id) + except Exception as e: + raise Exception(f'error: Cannot create new router: {e}') + except Exception as e: + raise e try: - router_id = ha_sql.create_ha_router(cluster_id) + vip = cluster.vip except Exception as e: - raise Exception(f'error: Cannot create new router: {e}') + raise Exception(f'Cannot get VIP: {e}') + try: + servers = _get_servers_dict(cluster) + except Exception as e: + raise Exception(f'Cannot get servers: {e}') try: vip_id = HaClusterVip.insert(cluster_id=cluster_id, router_id=router_id, vip=vip, return_master=cluster.return_master).execute() @@ -189,33 +192,27 @@ def insert_vip(cluster_id: int, cluster: HAClusterVIP, group_id: int) -> None: try: ha_sql.insert_or_update_slave(cluster_id, value['id'], value['eth'], value['master'], router_id) except Exception as e: - raise Exception(f'error: Cannot add server {value["ip"]}: {e}') + s = server_sql.get_server_by_id(value['id']) + raise Exception(f'error: Cannot add server {s.hostname}: {e}') if cluster.virt_server: add_or_update_virt(cluster, servers, cluster_id, vip_id, group_id) roxywi_common.logging(cluster_id, f'New cluster VIP: {vip} has been created', keep_history=1, roxywi=1, service='HA cluster') + return vip_id def update_slaves(servers: dict, cluster_id: int, router_id: int) -> None: - master_ip = None all_routers_in_cluster = HaClusterRouter.select(HaClusterRouter.id).where(HaClusterRouter.cluster_id == cluster_id).execute() server_ids_from_db = ha_sql.select_cluster_slaves(cluster_id, router_id) server_ids = [] server_ids_from_json = [] - for value in servers: - if value['master']: - master_ip = value['ip'] - for server in server_ids_from_db: server_ids.append(server[0]) for value in servers: - slave_id = value['id'] - if value['master']: - slave_id = server_sql.select_server_id_by_ip(master_ip) - server_ids_from_json.append(int(slave_id)) + server_ids_from_json.append(int(value['id'])) server_ids_for_deletion = set(server_ids) - set(server_ids_from_json) server_ids_for_adding = set(server_ids_from_json) - set(server_ids) @@ -228,7 +225,7 @@ def update_slaves(servers: dict, cluster_id: int, router_id: int) -> None: try: ha_sql.insert_or_update_slave(cluster_id, slave_id, value['eth'], value['master'], router) except Exception as e: - raise Exception(f'error: Cannot add new slave {value["name"]}: {e}') + raise Exception(f'error: Cannot add new slave {value["ip"]}: {e}') for o_s in server_ids_for_deletion: ha_sql.delete_master_from_slave(o_s) @@ -238,34 +235,19 @@ def update_slaves(servers: dict, cluster_id: int, router_id: int) -> None: except Exception as e: raise Exception(f'error: Cannot recreate slaves server: {e}') - for value in servers: - if value['master']: - continue - try: - ha_sql.update_server_master(master_ip, value['ip']) - except Exception as e: - raise Exception(f'error: Cannot update master on slave {value["ip"]}: {e}') - - for value in servers: - slave_id = value['id'] - if value['master']: - slave_id = server_sql.select_server_id_by_ip(master_ip) - try: - ha_sql.insert_or_update_slave(cluster_id, slave_id, value['eth'], value['master'], router_id) - except Exception as e: - raise Exception(f'error: Cannot update server {value["ip"]}: {e}') + _create_or_update_master_slaves_servers(cluster_id, servers, router_id) def add_or_update_virt(cluster: Union[HAClusterRequest, HAClusterVIP], servers: dict, cluster_id: int, vip_id: int, group_id: int) -> None: haproxy = 0 nginx = 0 apache = 0 - master_ip = None + master_id = None vip = str(cluster.vip) for value in servers: if value['master']: - master_ip = value['ip'] + master_id = value['id'] if ha_sql.check_ha_virt(vip_id): try: @@ -280,15 +262,34 @@ def add_or_update_virt(cluster: Union[HAClusterRequest, HAClusterVIP], servers: nginx = 1 if service.service_id == '2' else 0 apache = 1 if service.service_id == '4' else 0 try: - cred_id = ha_sql.get_cred_id_by_server_ip(master_ip) - firewall = 1 if server_mod.is_service_active(master_ip, 'firewalld') else 0 - ssh_settings = return_ssh_keys_path(master_ip) + server = server_sql.get_server_by_id(master_id) + c = ha_sql.get_cluster(cluster_id) virt_id = server_sql.add_server( - f'{vip}-VIP', vip, group_id, '1', '1', '0', cred_id, ssh_settings['port'], - f'VRRP IP for {cluster.name} cluster', haproxy, nginx, apache, firewall + f'{vip}-VIP', vip, group_id, '1', '1', '0', server.cred_id, server.port, + f'VRRP IP for {c.name} cluster', haproxy, nginx, apache, server.firewall_enable ) HaClusterVirt.insert(cluster_id=cluster_id, virt_id=virt_id, vip_id=vip_id).execute() roxywi_common.logging(cluster_id, f'New cluster virtual server for VIP: {vip} has been created', keep_history=1, roxywi=1, service='HA cluster') except Exception as e: roxywi_common.logging(cluster_id, f'error: Cannot create new cluster virtual server for VIP: {vip}: {e}', roxywi=1, service='HA cluster') + + +def _create_or_update_master_slaves_servers(cluster_id: int, servers: dict, router_id: int, create: bool = False) -> None: + for server in servers: + if server['master']: + continue + try: + ha_sql.update_master_server_by_slave_ip(server['id'], server['ip']) + except Exception as e: + raise Exception(f'error: Cannot update master on slave {server["ip"]: {e}}') + + for server in servers: + try: + ha_sql.insert_or_update_slave(cluster_id, server['id'], server['eth'], server['master'], router_id) + if create: + s = server_sql.get_server_by_id(server['id']) + roxywi_common.logging(cluster_id, f'New server {s.hostname} has been added to the cluster', keep_history=1, + roxywi=1, service='HA cluster') + except Exception as e: + raise Exception(f'error: Cannot update slave server {server["ip"]}: {e}') diff --git a/app/routes/ha/routes.py b/app/routes/ha/routes.py index 8c8bf83..cf7a41e 100644 --- a/app/routes/ha/routes.py +++ b/app/routes/ha/routes.py @@ -1,6 +1,7 @@ from flask import render_template, g, request from flask_jwt_extended import jwt_required +from app.modules.roxywi.exception import RoxywiResourceNotFound from app.routes.ha import bp from app.middleware import get_user_params, check_services import app.modules.db.ha_cluster as ha_sql @@ -25,11 +26,18 @@ def before_request(): @check_services @get_user_params() def get_ha_cluster(service, cluster_id): - router_id = ha_sql.get_router_id(cluster_id, default_router=1) + try: + router_id = ha_sql.get_router_id(cluster_id, default_router=1) + except RoxywiResourceNotFound: + router_id = None + if router_id: + slaves = ha_sql.select_cluster_slaves(cluster_id, router_id) + else: + slaves = {} kwargs = { 'servers': roxywi_common.get_dick_permit(virt=1), 'clusters': ha_sql.select_cluster(cluster_id), - 'slaves': ha_sql.select_cluster_slaves(cluster_id, router_id), + 'slaves': slaves, 'virts': ha_sql.select_clusters_virts(), 'vips': ha_sql.select_cluster_vips(cluster_id), 'cluster_services': ha_sql.select_cluster_services(cluster_id), @@ -103,15 +111,17 @@ def show_ha_cluster(service, cluster_id): return render_template('service.html', **kwargs) -@bp.route('//slaves/', methods=['GET', 'POST']) +@bp.route('//slaves//', methods=['GET', 'POST']) @check_services @get_user_params() -def get_slaves(service, cluster_id): +def get_slaves(service, cluster_id, vip_id): lang = g.user_params['lang'] if request.method == 'GET': router_id = ha_sql.get_router_id(cluster_id, default_router=1) else: - router_id = int(request.form.get('router_id')) + # router_id = int(request.form.get('router_id')) + vip = ha_sql.select_cluster_vip_by_vip_id(cluster_id, vip_id) + router_id = vip.router_id slaves = ha_sql.select_cluster_slaves(cluster_id, router_id) return render_template('ajax/ha/add_vip_slaves.html', lang=lang, slaves=slaves) diff --git a/app/static/css/styles.css b/app/static/css/styles.css index 3426ec5..5e9c4e9 100644 --- a/app/static/css/styles.css +++ b/app/static/css/styles.css @@ -186,7 +186,7 @@ pre { padding-top: 11px; text-align: right; margin-right: 20px; - width: 450px; + width: 460px; float: left; margin-left: 71%; margin-top: -50px; diff --git a/app/static/js/ha.js b/app/static/js/ha.js index 2f55db8..4c048f3 100644 --- a/app/static/js/ha.js +++ b/app/static/js/ha.js @@ -116,11 +116,11 @@ function createHaClusterStep1(edited=false, cluster_id=0, clean=true) { if (edited && clean) { let master_name = $('#master-server-'+cluster_id).text(); let master_ip = $('#master-ip-'+cluster_id).text(); + let master_id = $('#master-id-'+cluster_id).text(); $("#ha-cluster-master option").not(master_name).each(function (index) { $(this).prop('disabled', true); }); - $('#ha-cluster-master').append('').selectmenu("refresh"); - $('#ha-cluster-master').selectmenu("refresh"); + $('#ha-cluster-master').append('').selectmenu("refresh"); get_keepalived_ver($('#cur_master_ver'), master_ip); $.ajax({ url: api_prefix + "/ha/cluster/" + cluster_id, @@ -500,14 +500,14 @@ function increaseProgressValue(progress_step) { } $(progress_id).css('width', new_progress+'%'); } -function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edited=0) { +function add_vip_ha_cluster(cluster_id, cluster_name, vip_id='', vip='', edited=0) { let save_word = translate_div.attr('data-save'); let tabel_title = $("#add-vip-table").attr('title'); let buttons = []; let req_method = 'GET'; if (edited) { $.ajax({ - url: api_prefix + "/ha/cluster/" + cluster_id + "/vip/" + router_id, + url: api_prefix + "/ha/cluster/" + cluster_id + "/vip/" + vip_id, type: "GET", async: false, success: function (data) { @@ -544,7 +544,7 @@ function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edit if (!validateSlaves(jsonData)) { return false; } - saveVip(jsonData, cluster_id, $(this), cluster_name, edited, router_id, vip); + saveVip(jsonData, cluster_id, $(this), edited, vip_id); toastr.clear(); } }, { @@ -555,7 +555,7 @@ function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edit return false; } jsonData = createJsonVip('#vip_servers div span'); - saveVip(jsonData, cluster_id, $(this), cluster_name, edited, router_id, vip, true); + saveVip(jsonData, cluster_id, $(this), edited, vip_id, true); toastr.clear(); } }, { @@ -578,7 +578,7 @@ function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edit if (!validateSlaves(jsonData)) { return false; } - saveVip(jsonData, cluster_id, $(this), cluster_name, edited, router_id, vip); + saveVip(jsonData, cluster_id, $(this), edited, vip_id); toastr.clear(); } }, { @@ -591,10 +591,7 @@ function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edit }] } $.ajax({ - url: "/ha/cluster/slaves/" + cluster_id, - data: { - router_id: router_id, - }, + url: "/ha/cluster/slaves/" + cluster_id + "/" + vip_id, type: req_method, success: function (data) { if (data.indexOf('error:') != '-1') { @@ -627,7 +624,7 @@ function add_vip_ha_cluster(cluster_id, cluster_name, router_id='', vip='', edit }); dialog_div.dialog('open'); } -function saveVip(jsonData, cluster_id, dialog_id, cluster_name, edited, router_id='', vip='', deleted=false) { +function saveVip(jsonData, cluster_id, dialog_id, edited, vip_id='', deleted=false) { let req_type = 'POST' let return_master = 0 let virt_server = 0 @@ -645,15 +642,14 @@ function saveVip(jsonData, cluster_id, dialog_id, cluster_name, edited, router_i jsonData['return_master'] = return_master; jsonData['virt_server'] = virt_server; jsonData['use_src'] = use_src; - jsonData['name'] = cluster_name; let url = api_prefix + "/ha/cluster/" + cluster_id + "/vip"; if (edited) { req_type = 'PUT'; - jsonData['router_id'] = router_id; + url = api_prefix + "/ha/cluster/" + cluster_id + "/vip/" + vip_id; } if (deleted) { req_type = 'DELETE'; - url = api_prefix + "/ha/cluster/" + router_id + "/vip" + url = api_prefix + "/ha/cluster/" + cluster_id + "/vip/" + vip_id; } $.ajax({ url: url, @@ -759,18 +755,18 @@ function createJsonCluster(div_id) { let jsonData = {}; jsonData = {'servers': []}; jsonData['servers'].push({ - 'id': 1, + 'id': $('#ha-cluster-master option:selected').attr('data-id'), 'eth': $('#ha-cluster-master-interface').val(), - 'ip': $('#ha-cluster-master option:selected').val(), - 'name': $('#ha-cluster-master option:selected').text(), + // 'ip': $('#ha-cluster-master option:selected').val(), + // 'name': $('#ha-cluster-master option:selected').text(), 'master': 1 }); $(div_id).each(function () { let this_id = $(this).attr('id').split('-')[1]; let eth = $('#slave_int-' + this_id).val(); - let ip = $('#slave_int_div-' + this_id).attr('data-ip'); - let name = $('#slave_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); - jsonData['servers'].push({'id': this_id, 'eth': eth, 'ip': ip, 'name': name, 'master': 0}); + // let ip = $('#slave_int_div-' + this_id).attr('data-ip'); + // let name = $('#slave_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); + jsonData['servers'].push({'id': this_id, 'eth': eth, 'master': 0}); }); return jsonData; } @@ -780,21 +776,22 @@ function createJsonVip(div_id) { $(div_id).each(function () { let this_id = $(this).attr('id').split('-')[1]; let eth1 = $('#slave_int-' + this_id).val(); - let ip1 = $('#slave_int_div-' + this_id).attr('data-ip'); - let name1 = $('#slave_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); + // let ip1 = $('#slave_int_div-' + this_id).attr('data-ip'); + // let name1 = $('#slave_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); let eth = $('#master_int-' + this_id).val(); - let ip = $('#master_int_div-' + this_id).attr('data-ip'); - let name = $('#master_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); + // let ip = $('#master_int_div-' + this_id).attr('data-ip'); + // let name = $('#master_int_div-' + this_id).parent().text().replace('\n','').replace('\t','').trim(); if (eth) { - jsonData['servers'].push({'id': this_id, 'eth': eth, 'ip': ip, 'name': name, 'master': 1}); + jsonData['servers'].push({'id': this_id, 'eth': eth, 'master': 1}); } else { - jsonData['servers'].push({'id': this_id,'eth': eth1, 'ip': ip1, 'name': name1, 'master': 0}); + jsonData['servers'].push({'id': this_id,'eth': eth1, 'master': 0}); } }); return jsonData; } function validateSlaves(jsonData) { + console.log(jsonData) if (Object.keys(jsonData['servers']).length === 1) { toastr.error('error: There is must be at least one slave server'); return false; diff --git a/app/templates/ajax/ha/clusters.html b/app/templates/ajax/ha/clusters.html index ae58c4f..43b0024 100644 --- a/app/templates/ajax/ha/clusters.html +++ b/app/templates/ajax/ha/clusters.html @@ -21,6 +21,7 @@ {% if slave.31 %} Master name: {{ copy_to_clipboard(id='master-server-'+cluster.id|string(), value=slave.1) }}
Master IP: {{ copy_to_clipboard(id='master-ip-'+cluster.id|string(), value=slave.2) }}
+ {% endif %} {% endfor %} {{lang.words.slaves|title()}}: @@ -46,7 +47,7 @@ {%- for vip in vips %} {% if g.user_params['role'] <= 2 %} - {{vip.vip}} + {{vip.vip}} {% else %} {{vip.vip}} {%- endif -%} diff --git a/app/templates/include/add_backup.html b/app/templates/include/add_backup.html index cee9b10..90d641d 100644 --- a/app/templates/include/add_backup.html +++ b/app/templates/include/add_backup.html @@ -6,7 +6,7 @@ diff --git a/app/templates/include/admins_dialogs.html b/app/templates/include/admins_dialogs.html index fc4f622..0bac067 100644 --- a/app/templates/include/admins_dialogs.html +++ b/app/templates/include/admins_dialogs.html @@ -359,7 +359,9 @@ diff --git a/app/views/ha/views.py b/app/views/ha/views.py index 4d7ecd8..f39f013 100644 --- a/app/views/ha/views.py +++ b/app/views/ha/views.py @@ -9,9 +9,7 @@ import app.modules.roxywi.common as roxywi_common import app.modules.common.common as common import app.modules.service.ha_cluster as ha_cluster from app.middleware import get_user_params, page_for_admin, check_group, check_services -from app.modules.roxywi.class_models import ( - BaseResponse, IdResponse, HAClusterRequest, HAClusterVIP -) +from app.modules.roxywi.class_models import BaseResponse, IdResponse, HAClusterRequest, HAClusterVIP class HAView(MethodView): @@ -128,6 +126,9 @@ class HAView(MethodView): name: body required: true schema: + required: + - name + - services type: object properties: servers: @@ -140,8 +141,8 @@ class HAView(MethodView): type: 'string' master: type: 'integer' - name: - type: 'string' + id: + type: 'integer' name: type: string description: @@ -203,6 +204,9 @@ class HAView(MethodView): name: body required: true schema: + required: + - name + - services type: object properties: servers: @@ -215,8 +219,8 @@ class HAView(MethodView): type: 'string' master: type: 'integer' - name: - type: 'string' + id: + type: 'integer' name: type: string description: @@ -240,8 +244,6 @@ class HAView(MethodView): type: string docker: type: integer - router_id: - type: string responses: 201: description: HA cluster updated successfully @@ -299,7 +301,7 @@ class HAVIPView(MethodView): self.group_id = g.user_params['group_id'] @staticmethod - def get(service: str, cluster_id: int, router_id: int): + def get(service: str, cluster_id: int, vip_id: int): """ This endpoint retrieves information about the specified VIP. --- @@ -317,8 +319,8 @@ class HAVIPView(MethodView): required: true type: 'integer' - in: 'path' - name: 'router_id' - description: 'ID of the Router to retrieve' + name: 'vip_id' + description: 'ID of the VIP to retrieve' required: true type: 'integer' responses: @@ -328,29 +330,13 @@ class HAVIPView(MethodView): type: object properties: cluster_id: - type: 'object' - properties: - description: - type: 'string' - group_id: - type: 'integer' - id: - type: 'integer' - name: - type: 'string' - pos: - type: 'integer' - syn_flood: - type: 'integer' + type: 'integer' id: type: 'integer' return_master: type: 'integer' router_id: - type: 'object' - properties: - id: - type: 'integer' + type: 'integer' use_src: type: 'integer' vip: @@ -359,8 +345,8 @@ class HAVIPView(MethodView): description: Unexpected error """ try: - vip = ha_sql.select_cluster_vip(cluster_id, router_id) - settings = model_to_dict(vip) + vip = ha_sql.select_cluster_vip_by_vip_id(cluster_id, vip_id) + settings = model_to_dict(vip, recurse=False) is_virt = ha_sql.check_ha_virt(vip.id) settings.setdefault('virt_server', is_virt) return jsonify(settings) @@ -389,37 +375,29 @@ class HAVIPView(MethodView): name: body required: true schema: + required: + - servers + - vip type: object properties: servers: - type: object - additionalProperties: - type: object + type: 'array' + description: Must be at least 2 servers. One of them must be master = 1 + items: + type: 'object' properties: - eth: - type: string - ip: - type: string - name: - type: string + id: + type: 'integer' master: - type: integer - name: - type: string - description: - type: string + type: 'integer' vip: type: string virt_server: type: integer return_master: - type: string - syn_flood: type: integer use_src: type: integer - router_id: - type: string responses: 201: description: VIP created successfully @@ -431,14 +409,14 @@ class HAVIPView(MethodView): description: Unexpected error """ try: - ha_cluster.insert_vip(cluster_id, body, self.group_id) - return BaseResponse().model_dump(mode='json'), 201 + vip_id = ha_cluster.insert_vip(cluster_id, body, self.group_id) + return IdResponse(id=vip_id).model_dump(mode='json'), 201 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot create VIP') @validate(body=HAClusterVIP) - def put(self, service: str, cluster_id: int, body: HAClusterVIP): + def put(self, service: str, cluster_id: int, vip_id: int, body: HAClusterVIP): """ This endpoint allows to update a VIP for HA cluster. --- @@ -455,41 +433,38 @@ class HAVIPView(MethodView): description: 'ID of the HA cluster to update' required: true type: 'integer' + - in: 'path' + name: 'vip_id' + description: 'ID of the VIP to update' + required: true + type: 'integer' - in: body name: body required: true schema: + required: + - servers + - vip type: object properties: servers: - type: object - additionalProperties: - type: object + type: 'array' + description: Must be at least 2 servers. One of them must be master = 1 + items: + type: 'object' properties: - eth: - type: string - ip: - type: string - name: - type: string + id: + type: 'integer' master: - type: integer - name: - type: string - description: - type: string + type: 'integer' vip: type: string virt_server: type: integer return_master: type: string - syn_flood: - type: integer use_src: type: integer - router_id: - type: string responses: 201: description: VIP updated successfully @@ -500,14 +475,15 @@ class HAVIPView(MethodView): default: description: Unexpected error """ + vip = ha_sql.select_cluster_vip_by_vip_id(cluster_id, vip_id) try: - ha_cluster.update_vip(cluster_id, body.router_id, body, self.group_id) + ha_cluster.update_vip(cluster_id, vip.router_id, body, self.group_id) return BaseResponse().model_dump(mode='json'), 201 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot update VIP') @staticmethod - def delete(service: str, router_id: int): + def delete(service: str, cluster_id: int, vip_id: int): """ Delete a VIP --- @@ -520,7 +496,12 @@ class HAVIPView(MethodView): required: true type: 'string' - in: 'path' - name: 'router_id' + name: 'cluster_id' + description: 'ID of the HA cluster to delete VIP from' + required: true + type: 'integer' + - in: 'path' + name: 'vip_id' description: 'ID of the VIP to delete' required: true type: 'integer' @@ -532,11 +513,12 @@ class HAVIPView(MethodView): 404: description: VIP not found """ - router = ha_sql.get_router(router_id) + vip = ha_sql.select_cluster_vip_by_vip_id(cluster_id, vip_id) + router = ha_sql.get_router(vip.router_id) if router.default == 1: return roxywi_common.handler_exceptions_for_json_data(Exception(''), 'You cannot delete default VIP') try: - ha_sql.delete_ha_router(router_id) + ha_sql.delete_ha_router(vip.router_id) return BaseResponse().model_dump(mode='json'), 204 except Exception as e: return roxywi_common.handler_exceptions_for_json_data(e, 'Cannot delete VIP') @@ -592,10 +574,7 @@ class HAVIPsView(MethodView): return_master: type: 'integer' router_id: - type: 'object' - properties: - id: - type: 'integer' + type: 'integer' use_src: type: 'integer' vip: @@ -604,5 +583,5 @@ class HAVIPsView(MethodView): description: Unexpected error """ vips = ha_sql.select_cluster_vips(cluster_id) - vips = [model_to_dict(vip) for vip in vips] + vips = [model_to_dict(vip, recurse=False) for vip in vips] return jsonify(vips) diff --git a/app/views/server/backup_vews.py b/app/views/server/backup_vews.py index 823c3be..0583468 100644 --- a/app/views/server/backup_vews.py +++ b/app/views/server/backup_vews.py @@ -85,6 +85,14 @@ class BackupView(MethodView): description: The configuration for backup service schema: type: 'object' + required: + - cred_id + - rhost + - rpath + - server + - rserver + - time + - type properties: server: type: 'string' @@ -92,15 +100,19 @@ class BackupView(MethodView): rserver: type: 'string' description: 'The remote server where backup files should be stored' + example: 10.0.0.1 rpath: type: 'string' description: 'The path on the remote server where backup files should be stored' + example: /var/backup/ type: type: 'string' description: 'Type of the operation' + enum: [backup, synchronization] time: type: 'string' description: 'The timing for the backup task' + enum: [hourly, daily, weekly, monthly] cred_id: type: 'string' description: 'Credentials ID for the backup task' @@ -137,6 +149,14 @@ class BackupView(MethodView): description: The configuration for backup service schema: type: 'object' + required: + - cred_id + - rhost + - rpath + - server + - rserver + - time + - type properties: server: type: 'string' @@ -144,15 +164,19 @@ class BackupView(MethodView): rserver: type: 'string' description: 'The remote server where backup files should be stored' + example: 10.0.0.1 rpath: type: 'string' description: 'The path on the remote server where backup files should be stored' + example: /var/backup/ type: type: 'string' description: 'Type of the operation' + enum: [backup, synchronization] time: type: 'string' description: 'The timing for the backup task' + enum: [hourly, daily, weekly, monthly] cred_id: type: 'string' description: 'Credentials ID for the backup task' @@ -282,6 +306,13 @@ class S3BackupView(MethodView): description: The configuration for S3 backup service schema: type: 'object' + required: + - s3_server + - server + - bucket + - secret_key + - access_key + - time properties: s3_server: type: 'string' @@ -301,6 +332,7 @@ class S3BackupView(MethodView): time: type: 'string' description: 'The timing for the S3 backup task' + enum: [hourly, daily, weekly, monthly] description: type: 'string' description: 'Description for the S3 backup configuration' @@ -428,25 +460,37 @@ class GitBackupView(MethodView): description: The configuration for Git backup service schema: type: 'object' + required: + - cred_id + - server_id + - service_id + - init + - repo + - branch + - time properties: server_id: type: 'integer' description: 'The ID of the server to backed up' service_id: type: 'integer' - description: 'Service ID' + description: 'Service ID: 1: HAProxy, 2: NGINX, 3: Keepalived, 4: Apache' + example: 1 init: type: 'integer' description: 'Indicates whether to initialize the repository' repo: type: 'string' description: 'The repository from where to fetch the data for backup' + example: git@github.com:Example/haproxy_configs branch: type: 'string' description: 'The branch to pull for backup' + example: 'master' time: type: 'string' description: 'The timing for the Git backup task' + enum: [hourly, daily, weekly, monthly] cred_id: type: 'integer' description: 'The ID of the credentials to be used for backup' diff --git a/app/views/service/views.py b/app/views/service/views.py index f557a74..e979366 100644 --- a/app/views/service/views.py +++ b/app/views/service/views.py @@ -274,10 +274,10 @@ class ServiceConfigView(MethodView): required: true description: The ID or IP of the server - in: query - name: file_name + name: file_path type: 'string' required: false - description: The full path to the configuration file (used only for nginx and apache, replace "/" with 92) + description: The full path to the configuration file - in: query name: version type: 'string' @@ -289,10 +289,10 @@ class ServiceConfigView(MethodView): default: description: Unexpected error """ - if service in ('nginx', 'apache') and (query.file_name is None and query.version is None): + if service in ('nginx', 'apache') and (query.file_path is None and query.version is None): return ErrorResponse(error=f'There is must be "file_name" as query parameter for {service.title()}') - if query.file_name: - query.file_name = query.file_name.replace('/', '92') + if query.file_path: + query.file_path = query.file_path.replace('/', '92') try: server_ip = SupportClass(False).return_server_ip_or_id(server_id) diff --git a/config_other/requirements_deb.txt b/config_other/requirements_deb.txt deleted file mode 100644 index f907677..0000000 --- a/config_other/requirements_deb.txt +++ /dev/null @@ -1,23 +0,0 @@ -configparser>=3.5.0 -pytz>=2017.3 -tzlocal==2.0.0 -pyTelegramBotAPI>=3.6.3 -slack-sdk>=3.4.0 -distro>=1.2.0 -retry>=0.9.2 -psutil>=5.9.1 -pdpyras>=4.5.2 -Werkzeug==2.2.3 -Flask==2.2.5 -Flask-Login==0.6.2 -Flask-APScheduler==1.13.0 -Flask-Caching==2.1.0 -python3-nmap<=1.5.1 -aio-pika>=7.1.0 -pika>=1.2.0 -websockets>=9.0 -ansible-core>=2.11.12 -ansible-runner==2.3.2 -python-whois>=0.8.0 -requests==2.27.1 -netaddr>=0.10.1 diff --git a/config_other/requirements_el7.txt b/config_other/requirements_el7.txt deleted file mode 100644 index a5227db..0000000 --- a/config_other/requirements_el7.txt +++ /dev/null @@ -1,23 +0,0 @@ -pyTelegramBotAPI==3.6.3 -networkx==2.1 -matplotlib==2.1.2 -paramiko-ng>=2.5.0 -slack-sdk>=3.4.0 -peewee>=3.14.10 -PyMySQL>=1.0.2 -retry>=0.9.2 -pdpyras>=4.5.2 -tzlocal==2.0.0 -Werkzeug==2.0.3 -Flask==2.0.3 -Flask-APScheduler==1.12.4 -Flask-Caching==1.10.1 -Flask-Login==0.5.0 -python3-nmap<=1.5.1 -aio-pika>=7.1.0 -pika>=1.2.0 -websockets>=9.0 -ansible-core>=2.11.12 -ansible-runner==2.3.1 -python-whois>=0.8.0 -netaddr>=0.10.1 diff --git a/config_other/requirements_el8.txt b/config_other/requirements_el8.txt deleted file mode 100644 index c521fde..0000000 --- a/config_other/requirements_el8.txt +++ /dev/null @@ -1,24 +0,0 @@ -configparser==3.5.0 -pyTelegramBotAPI==3.6.3 -networkx>=3.3 -matplotlib>=2.1.2 -slack-sdk>=3.4.0 -peewee>=3.14.10 -PyMySQL>=1.0.2 -bottle>=0.12.18 -retry>=0.9.2 -tzlocal==2.0.0 -pdpyras>=4.5.2 -Werkzeug>=2.0.3 -Flask>=2.0.3 -Flask-APScheduler>=1.12.4 -Flask-Caching>=1.10.1 -Flask-Login>=0.5.0 -python3-nmap<=1.5.1 -aio-pika>=7.1.0 -pika>=1.2.0 -websockets>=9.0 -ansible-core>=2.11.12 -ansible-runner==2.3.1 -python-whois>=0.8.0 -netaddr>=0.10.1 diff --git a/config_other/requirements_el9.txt b/config_other/requirements_el9.txt deleted file mode 100644 index 27c0d66..0000000 --- a/config_other/requirements_el9.txt +++ /dev/null @@ -1,25 +0,0 @@ -configparser>=3.5.0 -pyTelegramBotAPI>=3.6.3 -networkx>=2.1 -matplotlib>=2.1.2 -slack-sdk>=3.4.0 -peewee>=3.14.10 -PyMySQL>=1.0.2 -bottle>=0.12.18 -retry>=0.9.2 -pdpyras>=4.5.2 -Werkzeug==2.2.3 -Flask==2.2.5 -Flask-Login==0.6.2 -Flask-APScheduler==1.13.0 -Flask-Caching==2.1.0 -python3-nmap<=1.5.1 -aio-pika>=7.1.0 -pika>=1.2.0 -websockets>=9.0 -tzlocal==2.0.0 -ansible-core>=2.11.12 -ansible-runner==2.3.1 -python-whois>=0.8.0 -requests==2.27.1 -netaddr>=0.10.1