diff --git a/app/create_db.py b/app/create_db.py index 51e9faf..da9190e 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -622,6 +622,33 @@ def update_db_v_8_0_2_1(): print("Updating... DB has been updated to version 8.0.2-1") +def update_db_v_8_1(): + try: + migrate( + migrator.rename_column('backups', 'server', 'server_id') + ) + except Exception as e: + if e.args[0] == 'no such column: "server"' or str(e) == '(1060, no such column: "server")': + print("Updating... DB has been updated to version 8.2") + elif e.args[0] == "'bool' object has no attribute 'sql'": + print("Updating... DB has been updated to version 8.2") + else: + print("An error occurred:", e) + +def update_db_v_8_1_0_1(): + try: + migrate( + migrator.rename_column('s3_backups', 'server', 'server_id') + ) + except Exception as e: + if e.args[0] == 'no such column: "server"' or str(e) == '(1060, no such column: "server")': + print("Updating... DB has been updated to version 8.2") + elif e.args[0] == "'bool' object has no attribute 'sql'": + print("Updating... DB has been updated to version 8.2") + else: + print("An error occurred:", e) + + def update_ver(): try: Version.update(version='8.1.0').execute() @@ -650,4 +677,6 @@ def update_all(): update_db_v_8() update_db_v_8_0_2() update_db_v_8_0_2_1() + update_db_v_8_1() + update_db_v_8_1_0_1() update_ver() diff --git a/app/modules/db/backup.py b/app/modules/db/backup.py index 039be83..62aa38c 100644 --- a/app/modules/db/backup.py +++ b/app/modules/db/backup.py @@ -2,11 +2,17 @@ from app.modules.db.db_model import Backup, S3Backup, GitSetting from app.modules.db.common import out_error from app.modules.roxywi.exception import RoxywiResourceNotFound +models = { + 'fs': Backup, + 's3': S3Backup, + 'git': GitSetting, + } + -def insert_backup_job(server, rserver, rpath, backup_type, time, cred, description): +def insert_backup_job(server_id, rserver, rpath, backup_type, time, cred, description): try: return Backup.insert( - server=server, rhost=rserver, rpath=rpath, type=backup_type, time=time, + server_id=server_id, rhost=rserver, rpath=rpath, type=backup_type, time=time, cred_id=cred, description=description ).execute() except Exception as e: @@ -20,32 +26,31 @@ def insert_s3_backup_job(**kwargs): out_error(e) -def update_backup(server, rserver, rpath, backup_type, time, cred, description, backup_id): - backup_update = Backup.update( - server=server, rhost=rserver, rpath=rpath, type=backup_type, time=time, - cred_id=cred, description=description - ).where(Backup.id == backup_id) +def update_s3_backup_job(backup_id: int, model: str, **kwargs): + model = models[model] try: - backup_update.execute() + return model.update(**kwargs).where(model.id == backup_id).execute() except Exception as e: out_error(e) -def delete_backups(backup_id: int) -> None: +def update_backup(server_id, rserver, rpath, backup_type, time, cred, description, backup_id): + backup_update = Backup.update( + server_id=server_id, rhost=rserver, rpath=rpath, type=backup_type, time=time, + cred_id=cred, description=description + ).where(Backup.id == backup_id) try: - Backup.delete().where(Backup.id == backup_id).execute() + backup_update.execute() except Exception as e: out_error(e) -def delete_s3_backups(backup_id: int) -> bool: +def delete_backup(backup_id: int, model: str) -> None: + model = models[model] try: - S3Backup.delete().where(S3Backup.id == backup_id).execute() + model.delete().where(model.id == backup_id).execute() except Exception as e: out_error(e) - return False - else: - return True def insert_new_git(server_id, service_id, repo, branch, period, cred, description) -> int: @@ -58,13 +63,6 @@ def insert_new_git(server_id, service_id, repo, branch, period, cred, descriptio out_error(e) -def delete_git(git_id: int) -> None: - try: - GitSetting.delete().where(GitSetting.id == git_id).execute() - except Exception as e: - out_error(e) - - def select_gits(**kwargs): if kwargs.get("server_id") is not None and kwargs.get("service_id") is not None: query = GitSetting.select().where( @@ -81,8 +79,8 @@ def select_gits(**kwargs): def select_backups(**kwargs): - if kwargs.get("server") is not None and kwargs.get("rserver") is not None: - query = Backup.select().where((Backup.server == kwargs.get("server")) & (Backup.rhost == kwargs.get("rserver"))) + if kwargs.get("backup_id") is not None: + query = Backup.select().where(Backup.id == kwargs.get("backup_id")) else: query = Backup.select().order_by(Backup.id) @@ -110,21 +108,10 @@ def select_s3_backups(**kwargs): return query_res -def check_exists_backup(server: str) -> bool: - try: - backup = Backup.get(Backup.server == server) - except Exception: - pass - else: - if backup.id is not None: - return True - else: - return False - - -def check_exists_s3_backup(server: str) -> bool: +def check_exists_backup(server_id: int, model: str) -> bool: + model = models[model] try: - backup = S3Backup.get(S3Backup.server == server) + backup = model.get(model.server_id == server_id) except Exception: pass else: @@ -135,11 +122,6 @@ def check_exists_s3_backup(server: str) -> bool: def get_backup(backup_id: int, model: str) -> Backup: - models = { - 'fs': Backup, - 's3': S3Backup, - 'git': GitSetting, - } model = models[model] try: return model.get(model.id == backup_id) diff --git a/app/modules/db/db_model.py b/app/modules/db/db_model.py index b7f03fa..6970b9f 100644 --- a/app/modules/db/db_model.py +++ b/app/modules/db/db_model.py @@ -210,7 +210,7 @@ class Cred(BaseModel): class Backup(BaseModel): id = AutoField() - server = CharField() + server_id = CharField() rhost = CharField() rpath = CharField() type = CharField(column_name='type') @@ -224,7 +224,7 @@ class Backup(BaseModel): class S3Backup(BaseModel): id = AutoField() - server = CharField() + server_id = CharField() s3_server = CharField() bucket = CharField() secret_key = CharField() diff --git a/app/modules/roxywi/class_models.py b/app/modules/roxywi/class_models.py index e64ddc8..10cfad8 100644 --- a/app/modules/roxywi/class_models.py +++ b/app/modules/roxywi/class_models.py @@ -4,7 +4,7 @@ from typing import Optional, Annotated, Union, Literal, Any, Dict, List from shlex import quote from pydantic_core import CoreSchema, core_schema -from pydantic import BaseModel, Base64Str, StringConstraints, IPvAnyAddress, GetCoreSchemaHandler +from pydantic import BaseModel, Base64Str, StringConstraints, IPvAnyAddress, GetCoreSchemaHandler, AnyUrl DomainName = Annotated[str, StringConstraints(pattern=r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,61}[a-z0-9]$")] @@ -251,21 +251,21 @@ class SettingsRequest(BaseModel): class BackupRequest(BaseModel): cred_id: int - server: Union[IPvAnyAddress, DomainName] + server_id: int rserver: Optional[Union[IPvAnyAddress, DomainName]] = None description: Optional[EscapedString] = None rpath: Optional[EscapedString] = None - type: Optional[EscapedString] = None - time: Optional[EscapedString] = None + type: Literal['backup', 'synchronization'] = None + time: Literal['hourly', 'daily', 'weekly', 'monthly'] = None class S3BackupRequest(BaseModel): - server: Union[IPvAnyAddress, DomainName] - s3_server: Optional[Union[IPvAnyAddress, DomainName]] = None + server_id: int + s3_server: Optional[Union[IPvAnyAddress, AnyUrl]] = None bucket: EscapedString secret_key: Optional[EscapedString] = None access_key: Optional[EscapedString] = None - time: Optional[EscapedString] = None + time: Literal['hourly', 'daily', 'weekly', 'monthly'] = None description: Optional[EscapedString] = None diff --git a/app/modules/roxywi/common.py b/app/modules/roxywi/common.py index 18fcf46..933bb0c 100644 --- a/app/modules/roxywi/common.py +++ b/app/modules/roxywi/common.py @@ -17,7 +17,7 @@ import app.modules.db.ha_cluster as ha_sql import app.modules.roxy_wi_tools as roxy_wi_tools from app.modules.roxywi.class_models import ErrorResponse from app.modules.roxywi.exception import RoxywiResourceNotFound, RoxywiGroupMismatch, RoxywiGroupNotFound, \ - RoxywiPermissionError + RoxywiPermissionError, RoxywiConflictError get_config_var = roxy_wi_tools.GetConfigVar() @@ -337,5 +337,7 @@ def handler_exceptions_for_json_data(ex: Exception, main_ex_mes: str) -> tuple[d return handle_json_exceptions(ex, 'Resource not found in group'), 404 elif isinstance(ex, RoxywiPermissionError): return handle_json_exceptions(ex, 'You cannot edit this resource'), 403 + elif isinstance(ex, RoxywiConflictError): + return handle_json_exceptions(ex, 'Conflict'), 429 else: return handle_json_exceptions(ex, main_ex_mes), 500 diff --git a/app/modules/server/ssh.py b/app/modules/server/ssh.py index 90c3c55..e41deca 100644 --- a/app/modules/server/ssh.py +++ b/app/modules/server/ssh.py @@ -19,10 +19,10 @@ error_mess = common.error_mess get_config = roxy_wi_tools.GetConfigVar() -def return_ssh_keys_path(server_ip: str, **kwargs) -> dict: +def return_ssh_keys_path(server_ip: str, cred_id: int = None) -> dict: ssh_settings = {} - if kwargs.get('id'): - sshs = cred_sql.select_ssh(id=kwargs.get('id')) + if cred_id: + sshs = cred_sql.select_ssh(id=cred_id) else: sshs = cred_sql.select_ssh(serv=server_ip) diff --git a/app/modules/service/backup.py b/app/modules/service/backup.py index 33897c8..aef5f1a 100644 --- a/app/modules/service/backup.py +++ b/app/modules/service/backup.py @@ -1,4 +1,3 @@ -from docutils.parsers.rst.directives import body from flask import render_template import app.modules.db.sql as sql @@ -10,15 +9,17 @@ import app.modules.server.ssh as ssh_mod import app.modules.roxywi.common as roxywi_common import app.modules.service.installation as installation_mod from app.modules.roxywi.class_models import BackupRequest, IdResponse, IdDataResponse, BaseResponse, S3BackupRequest, GitBackupRequest +from app.modules.roxywi.exception import RoxywiConflictError def create_backup_inv(json_data: BackupRequest, del_id: int = 0) -> None: - ssh_settings = ssh_mod.return_ssh_keys_path(str(json_data.server), id=json_data.cred_id) + server = server_sql.get_server_by_id(json_data.server_id) + ssh_settings = ssh_mod.return_ssh_keys_path(server.ip, json_data.cred_id) inv = {"server": {"hosts": {}}} server_ips = [] - inv['server']['hosts'][str(json_data.server)] = { + inv['server']['hosts'][server.ip] = { 'HOST': str(json_data.rserver), - "SERVER": str(json_data.server), + "SERVER": server.ip, "TYPE": json_data.type, "TIME": json_data.time, "RPATH": json_data.rpath, @@ -26,7 +27,7 @@ def create_backup_inv(json_data: BackupRequest, del_id: int = 0) -> None: "USER": ssh_settings['user'], "KEY": ssh_settings['key'] } - server_ips.append(str(json_data.server)) + server_ips.append(str(server.ip)) try: installation_mod.run_ansible(inv, server_ips, 'backup') @@ -35,9 +36,10 @@ def create_backup_inv(json_data: BackupRequest, del_id: int = 0) -> None: def create_s3_backup_inv(data: S3BackupRequest, tag: str) -> None: + server = server_sql.get_server_by_id(data.server_id) inv = {"server": {"hosts": {}}} inv["server"]["hosts"]["localhost"] = { - "SERVER": str(data.server), + "SERVER": server.hostname, "S3_SERVER": str(data.s3_server), "BUCKET": data.bucket, "SECRET_KEY": data.secret_key, @@ -54,7 +56,7 @@ def create_s3_backup_inv(data: S3BackupRequest, tag: str) -> None: def create_git_backup_inv(data: GitBackupRequest, server_ip: str, service: str, del_job: int = 0) -> None: service_config_dir = sql.get_setting(service + '_dir') - ssh_settings = ssh_mod.return_ssh_keys_path(server_ip, id=data.cred_id) + ssh_settings = ssh_mod.return_ssh_keys_path(server_ip, data.cred_id) inv = {"server": {"hosts": {}}} inv["server"]["hosts"][server_ip] = { "REPO": data.repo, @@ -74,43 +76,44 @@ def create_git_backup_inv(data: GitBackupRequest, server_ip: str, service: str, def create_backup(json_data: BackupRequest, is_api: bool) -> tuple: - if backup_sql.check_exists_backup(json_data.server): - raise Exception(f'warning: Backup job for {json_data.server} already exists') + if backup_sql.check_exists_backup(json_data.server_id, 'fs'): + raise RoxywiConflictError('FS backup for this server already exists') create_backup_inv(json_data) - last_id = backup_sql.insert_backup_job(json_data.server, json_data.rserver, json_data.rpath, json_data.type, + last_id = backup_sql.insert_backup_job(json_data.server_id, json_data.rserver, json_data.rpath, json_data.type, json_data.time, json_data.cred_id, json_data.description) - roxywi_common.logging('backup ', f' a new backup job for server {json_data.server} has been created', roxywi=1, login=1) + roxywi_common.logging('backup ', f'A new backup job for server {json_data.server_id} has been created', roxywi=1, login=1) if is_api: return IdResponse(id=last_id).model_dump(mode='json'), 201 else: data = render_template( 'ajax/new_backup.html', - backups=backup_sql.select_backups(server=json_data.server, rserver=json_data.rserver), - sshs=cred_sql.select_ssh() + backups=backup_sql.select_backups(backup_id=last_id), + sshs=cred_sql.select_ssh(), + servers=roxywi_common.get_dick_permit(virt=1, disable=0, only_group=1), ) return IdDataResponse(data=data, id=last_id).model_dump(mode='json'), 201 def delete_backup(json_data: BackupRequest, backup_id: int) -> tuple: - create_backup_inv(json_data, backup_id) - backup_sql.delete_backups(backup_id) - roxywi_common.logging('backup ', f' a backup job for server {json_data.server} has been deleted', roxywi=1, login=1) + create_backup_inv(json_data, 1) + backup_sql.delete_backup(backup_id, 'fs') + roxywi_common.logging('backup ', f'A backup job for server {json_data.server_id} has been deleted', roxywi=1, login=1) return BaseResponse().model_dump(mode='json'), 204 def update_backup(json_data: BackupRequest, backup_id: int) -> tuple: create_backup_inv(json_data) - backup_sql.update_backup(json_data.server, json_data.rserver, json_data.rpath, json_data.type, + backup_sql.update_backup(json_data.server_id, json_data.rserver, json_data.rpath, json_data.type, json_data.time, json_data.cred_id, json_data.description, backup_id) - roxywi_common.logging('backup ', f' a backup job for server {json_data.server} has been updated', roxywi=1, login=1) + roxywi_common.logging('backup ', f'A backup job for server {json_data.server_id} has been updated', roxywi=1, login=1) return BaseResponse().model_dump(mode='json'), 201 def create_s3_backup(data: S3BackupRequest, is_api: bool) -> tuple: - if backup_sql.check_exists_s3_backup(data.server): - raise Exception(f'Backup job for {data.server} already exists') + if backup_sql.check_exists_backup(data.server_id, 's3'): + raise RoxywiConflictError('S3 backup for this server already exists') try: create_s3_backup_inv(data, 'add') @@ -119,7 +122,7 @@ def create_s3_backup(data: S3BackupRequest, is_api: bool) -> tuple: try: last_id = backup_sql.insert_s3_backup_job(**data.model_dump(mode='json')) - roxywi_common.logging('backup ', f'a new S3 backup job for server {data.server} has been created', roxywi=1, login=1) + roxywi_common.logging('backup ', f'A new S3 backup job for server {data.server_id} has been created', roxywi=1, login=1) except Exception as e: raise Exception(e) @@ -133,8 +136,8 @@ def create_s3_backup(data: S3BackupRequest, is_api: bool) -> tuple: def delete_s3_backup(data: S3BackupRequest, backup_id: int) -> None: try: create_s3_backup_inv(data, 'delete') - backup_sql.delete_s3_backups(backup_id) - roxywi_common.logging('backup ', f'a S3 backup job for server {data.server} has been deleted', roxywi=1, login=1) + backup_sql.delete_backup(backup_id, 's3') + roxywi_common.logging('backup ', f'The S3 backup job for server {data.server_id} has been deleted', roxywi=1, login=1) except Exception as e: raise e @@ -180,7 +183,7 @@ def delete_git_backup(data: GitBackupRequest, backup_id: int) -> tuple: raise Exception(e) try: - backup_sql.delete_git(backup_id) + backup_sql.delete_backup(backup_id, 'git') except Exception as e: raise Exception(e) diff --git a/app/scripts/ansible/roles/backup.yml b/app/scripts/ansible/roles/backup.yml index 1645dc1..968967b 100644 --- a/app/scripts/ansible/roles/backup.yml +++ b/app/scripts/ansible/roles/backup.yml @@ -8,19 +8,15 @@ path: "{{ RPATH }}/roxy-wi-configs-backup/configs" state: directory owner: "{{ ansible_user }}" - when: not DELJOB -- hosts: localhost - become: yes - become_method: sudo - gather_facts: no - tasks: + when: not DELJOB and ansible_host != "localhost" - name: Creates backup jobs cron: name: "Roxy-WI Backup configs for server {{ SERVER }} {{ item }}" special_time: "{{ TIME }}" - job: "rsync -arv {{ TYPE }} /var/lib/roxy-wi/configs/{{ item }}/{{ SERVER }}* {{ ansible_user }}@{{ HOST }}:{{ RPATH }}/roxy-wi-configs-backup/configs/{{ item }} -e 'ssh -i {{ KEY }} -o StrictHostKeyChecking=no' --log-file=/var/www/haproxy-wi/log/backup.log" + job: "rsync -arv {{ TYPE }} /var/lib/roxy-wi/configs/{{ item }}/{{ SERVER }}* {{ USER }}@{{ HOST }}:{{ RPATH }}/roxy-wi-configs-backup/configs/{{ item }} -e 'ssh -i {{ KEY }} -o StrictHostKeyChecking=no' --log-file=/var/www/haproxy-wi/log/backup.log" when: not DELJOB + delegate_to: localhost with_items: - kp_config - hap_config @@ -32,6 +28,7 @@ name: "Roxy-WI Backup configs for server {{ SERVER }} {{ item }}" state: absent when: DELJOB + delegate_to: localhost with_items: - kp_config - hap_config diff --git a/app/static/js/backup.js b/app/static/js/backup.js index 2e309aa..168401d 100644 --- a/app/static/js/backup.js +++ b/app/static/js/backup.js @@ -139,7 +139,7 @@ function addBackup(dialog_id) { valid = valid && checkLength($('#backup-credentials'), "backup credentials", 1); if (valid) { let jsonData = { - "server": $('#backup-server').val(), + "server_id": $('#backup-server').val(), "rserver": $('#rserver').val(), "rpath": $('#rpath').val(), "type": $('#backup-type').val(), @@ -175,8 +175,8 @@ function addS3Backup(dialog_id) { valid = valid && checkLength($('#s3_access_key'), "S3 access key", 1); if (valid) { let json_data = { - "s3_server": $('#s3-backup-server').val(), - "server": $('#s3_server').val(), + "s3_server": $('#s3_server').val(), + "server_id": $('#s3-backup-server').val(), "bucket": $('#s3_bucket').val(), "secret_key": $('#s3_secret_key').val(), "access_key": $('#s3_access_key').val(), @@ -332,7 +332,7 @@ function removeBackup(id) { $("#backup-table-" + id).css("background-color", "#f2dede"); let jsonData = { "cred_id": $('#backup-credentials-' + id).val(), - "server": $('#backup-server-' + id).text(), + "server_id": $('#backup-server-id-' + id).val(), } $.ajax({ url: api_prefix + "/server/backup/fs/" + id, @@ -360,7 +360,7 @@ function removeS3Backup(id) { $("#backup-table-s3-" + id).css("background-color", "#f2dede"); let jsonData = { "bucket": $('#bucket-' + id).text(), - "server": $('#backup-s3-server-' + id).text(), + "server_id": $('#backup-s3-server-' + id).text(), } $.ajax({ url: api_prefix + "/server/backup/s3/" + id, @@ -419,7 +419,7 @@ function updateBackup(id) { toastr.error('All fields must be completed'); } else { let jsonData = { - "server": $('#backup-server-' + id).text(), + "server_id": $('#backup-server-id-' + id).val(), "rserver": $('#backup-rserver-' + id).val(), "rpath": $('#backup-rpath-' + id).val(), "type": $('#backup-type-' + id).val(), diff --git a/app/templates/ajax/new_backup.html b/app/templates/ajax/new_backup.html index 32711cb..c897a28 100644 --- a/app/templates/ajax/new_backup.html +++ b/app/templates/ajax/new_backup.html @@ -1,77 +1,54 @@ {% for b in backups %} -