github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: Account backup report

pull/14579/head
feng 2024-12-04 16:28:49 +08:00 committed by feng626
parent 9598174745
commit f9501840cd
12 changed files with 152 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/accounts/api/automations/backup.py
View File

@ -9,11 +9,11 @@ from orgs.mixins.api import OrgBulkModelViewSet
from .base import AutomationExecutionViewSet
__all__ = [
'AccountBackupPlanViewSet', 'BackupAccountExecutionViewSet'
'BackupAccountViewSet', 'BackupAccountExecutionViewSet'
]
class AccountBackupPlanViewSet(OrgBulkModelViewSet):
class BackupAccountViewSet(OrgBulkModelViewSet):
model = BackupAccountAutomation
filterset_fields = ('name',)
search_fields = filterset_fields
@ -21,8 +21,13 @@ class AccountBackupPlanViewSet(OrgBulkModelViewSet):
class BackupAccountExecutionViewSet(AutomationExecutionViewSet):
serializer_class = serializers.BackupAccountExecutionSerializer
http_method_names = ['get', 'post', 'options']
rbac_perms = (
("list", "accounts.view_backupaccountexecution"),
("retrieve", "accounts.view_backupaccountexecution"),
("create", "accounts.add_backupaccountexecution"),
("report", "accounts.view_backupaccountexecution"),
)
tp = AutomationTypes.backup_account
def get_queryset(self):

91
apps/accounts/automations/backup_account/handlers.py
View File

@ -3,15 +3,17 @@ import time
from collections import defaultdict, OrderedDict
from django.conf import settings
from django.db.models import F
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from xlsxwriter import Workbook
from accounts.const import AccountBackupType
from accounts.models.automations.backup_account import BackupAccountAutomation
from accounts.models import BackupAccountAutomation, Account
from accounts.notifications import AccountBackupExecutionTaskMsg, AccountBackupByObjStorageExecutionTaskMsg
from accounts.serializers import AccountSecretSerializer
from assets.const import AllTypes
from common.const import Status
from common.utils.file import encrypt_and_compress_zip_file, zip_files
from common.utils.timezone import local_now_filename, local_now_display
from terminal.models.component.storage import ReplayStorage
@ -74,9 +76,9 @@ class BaseAccountHandler:
class AssetAccountHandler(BaseAccountHandler):
@staticmethod
def get_filename(plan_name):
def get_filename(name):
filename = os.path.join(
PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.xlsx'
PATH, f'{name}-{local_now_filename()}-{time.time()}.xlsx'
)
return filename
@ -118,32 +120,41 @@ class AssetAccountHandler(BaseAccountHandler):
cls.handler_secret(data, section)
data_map.update(cls.add_rows(data, header_fields, sheet_name))
number_of_backup_accounts = _('Number of backup accounts')
print('\n\033[33m- {}: {}\033[0m'.format(number_of_backup_accounts, accounts.count()))
print('\033[33m- {}: {}\033[0m'.format(number_of_backup_accounts, accounts.count()))
return data_map
class AccountBackupHandler:
def __init__(self, execution):
def __init__(self, manager, execution):
self.manager = manager
self.execution = execution
self.plan_name = self.execution.plan.name
self.is_frozen = False # 任务状态冻结标志
self.name = self.execution.snapshot.get('name', '-')
def get_accounts(self):
# TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作
types = self.execution.snapshot.get('types', [])
self.manager.summary['total_types'] = len(types)
qs = Account.objects.filter(
asset__platform__type__in=types
).annotate(type=F('asset__platform__type'))
return qs
def create_excel(self, section='complete'):
hint = _('Generating asset or application related backup information files')
hint = _('Generating asset related backup information files')
print(
'\n'
f'\033[32m>>> {hint}\033[0m'
''
)
# Print task start date
time_start = time.time()
files = []
accounts = self.execution.backup_accounts
accounts = self.get_accounts()
self.manager.summary['total_accounts'] = accounts.count()
data_map = AssetAccountHandler.create_data_map(accounts, section)
if not data_map:
return files
filename = AssetAccountHandler.get_filename(self.plan_name)
filename = AssetAccountHandler.get_filename(self.name)
wb = Workbook(filename)
for sheet, data in data_map.items():
@ -164,19 +175,18 @@ class AccountBackupHandler:
return
recipients = User.objects.filter(id__in=list(recipients))
print(
'\n'
f'\033[32m>>> {_("Start sending backup emails")}\033[0m'
''
)
plan_name = self.plan_name
name = self.name
for user in recipients:
if not user.secret_key:
attachment_list = []
else:
attachment = os.path.join(PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.zip')
attachment = os.path.join(PATH, f'{name}-{local_now_filename()}-{time.time()}.zip')
encrypt_and_compress_zip_file(attachment, user.secret_key, files)
attachment_list = [attachment, ]
AccountBackupExecutionTaskMsg(plan_name, user).publish(attachment_list)
attachment_list = [attachment]
AccountBackupExecutionTaskMsg(name, user).publish(attachment_list)
for file in files:
os.remove(file)
@ -186,49 +196,37 @@ class AccountBackupHandler:
return
recipients = ReplayStorage.objects.filter(id__in=list(recipients))
print(
'\n'
'\033[32m>>> 📃 ---> sftp \033[0m'
''
)
plan_name = self.plan_name
name = self.name
encrypt_file = _('Encrypting files using encryption password')
for rec in recipients:
attachment = os.path.join(PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.zip')
attachment = os.path.join(PATH, f'{name}-{local_now_filename()}-{time.time()}.zip')
if password:
print(f'\033[32m>>> {encrypt_file}\033[0m')
encrypt_and_compress_zip_file(attachment, password, files)
else:
zip_files(attachment, files)
attachment_list = attachment
AccountBackupByObjStorageExecutionTaskMsg(plan_name, rec).publish(attachment_list)
AccountBackupByObjStorageExecutionTaskMsg(name, rec).publish(attachment_list)
file_sent_to = _('The backup file will be sent to')
print('{}: {}({})'.format(file_sent_to, rec.name, rec.id))
for file in files:
os.remove(file)
def step_perform_task_update(self, is_success, reason):
self.execution.reason = reason[:1024]
self.execution.is_success = is_success
self.execution.save()
def _run(self):
is_success = False
error = '-'
try:
backup_type = self.execution.snapshot.get('backup_type', AccountBackupType.email.value)
if backup_type == AccountBackupType.email.value:
backup_type = self.execution.snapshot.get('backup_type', AccountBackupType.email)
if backup_type == AccountBackupType.email:
self.backup_by_email()
elif backup_type == AccountBackupType.object_storage.value:
elif backup_type == AccountBackupType.object_storage:
self.backup_by_obj_storage()
except Exception as e:
self.is_frozen = True
print(e)
error = str(e)
else:
is_success = True
finally:
reason = error
self.step_perform_task_update(is_success, reason)
print(f'\033[31m>>> {error}\033[0m')
self.execution.status = Status.error
self.execution.summary['error'] = error
def backup_by_obj_storage(self):
object_id = self.execution.snapshot.get('id')
@ -265,7 +263,7 @@ class AccountBackupHandler:
f'\033[31m>>> {warn_text}\033[0m'
''
)
raise RecipientsNotFound('Not Found Recipients')
return
if recipients_part_one and recipients_part_two:
print(f'\033[32m>>> {split_help_text}\033[0m')
files = self.create_excel(section='front')
@ -279,16 +277,5 @@ class AccountBackupHandler:
self.send_backup_mail(files, recipients)
def run(self):
plan_start = _('Plan start')
time_cost = _('Duration')
error = _('An exception occurred during task execution')
print('{}: {}'.format(plan_start, local_now_display()))
time_start = time.time()
try:
self._run()
except Exception as e:
print(error)
print(e)
finally:
timedelta = round((time.time() - time_start), 2)
print('{}: {}s'.format(time_cost, timedelta))
print('{}: {}'.format(_('Plan start'), local_now_display()))
self._run()

16
apps/accounts/automations/backup_account/manager.py
View File

@ -1,11 +1,9 @@
# -*- coding: utf-8 -*-
#
import time
from django.utils.translation import gettext_lazy as _
from assets.automations.base.manager import BaseManager
from common.db.utils import safe_db_connection
from common.utils.timezone import local_now_display
from .handlers import AccountBackupHandler
@ -14,23 +12,19 @@ class AccountBackupManager(BaseManager):
def do_run(self):
execution = self.execution
account_backup_execution_being_executed = _('The account backup plan is being executed')
print(f'\n\033[33m# {account_backup_execution_being_executed}\033[0m')
handler = AccountBackupHandler(execution)
print(f'\033[33m# {account_backup_execution_being_executed}\033[0m')
handler = AccountBackupHandler(self, execution)
handler.run()
def send_report_if_need(self):
pass
def update_execution(self):
timedelta = int(time.time() - self.time_start)
self.execution.timedelta = timedelta
with safe_db_connection():
self.execution.save(update_fields=['timedelta', ])
def print_summary(self):
print('\n\n' + '-' * 80)
plan_execution_end = _('Plan execution end')
print('{} {}\n'.format(plan_execution_end, local_now_display()))
time_cost = _('Duration')
print('{}: {}s'.format(time_cost, self.duration))
def get_report_template(self):
return "accounts/backup_account_report.html"

2
apps/accounts/automations/endpoint.py
View File

@ -18,7 +18,7 @@ class ExecutionManager:
AutomationTypes.gather_accounts: GatherAccountsManager,
AutomationTypes.verify_gateway_account: VerifyGatewayAccountManager,
AutomationTypes.check_account: CheckAccountManager,
'backup_account': AccountBackupManager,
AutomationTypes.backup_account: AccountBackupManager,
}
def __init__(self, execution):

32
apps/accounts/models/automations/backup_account.py
View File

@ -79,35 +79,3 @@ class BackupAccountAutomation(AccountBaseAutomation):
def save(self, *args, **kwargs):
self.type = AutomationTypes.backup_account
super().save(*args, **kwargs)
# class AccountBackupExecution(AutomationExecution):
# plan = models.ForeignKey(
# 'AccountBackupAutomation', related_name='execution', on_delete=models.CASCADE,
# verbose_name=_('Account backup plan')
# )
#
# class Meta:
# verbose_name = _('Account backup execution')
#
# @property
# def types(self):
# types = self.snapshot.get('types')
# return types
#
# @lazyproperty
# def backup_accounts(self):
# from accounts.models import Account
# # TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作
# qs = Account.objects.filter(
# asset__platform__type__in=self.types
# ).annotate(type=F('asset__platform__type'))
# return qs
#
# @property
# def manager_type(self):
# return 'backup_account'
#
# def start(self):
# from accounts.automations.endpoint import ExecutionManager
# manager = ExecutionManager(execution=self)
# return manager.run()

3
apps/accounts/models/automations/base.py
View File

@ -40,6 +40,9 @@ class AutomationExecution(AssetAutomationExecution):
('view_pushaccountexecution', _('Can view push account execution')),
('add_pushaccountexecution', _('Can add push account execution')),
('view_backupaccountexecution', _('Can view backup account execution')),
('add_backupaccountexecution', _('Can add backup account execution')),
]
@property

21
apps/accounts/serializers/automations/backup.py
View File

@ -1,17 +1,16 @@
# -*- coding: utf-8 -*-
#
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from accounts.const import AutomationTypes
from accounts.models import BackupAccountAutomation
from common.const.choices import Trigger
from common.serializers.fields import LabeledChoiceField, EncryptedField
from common.serializers.fields import EncryptedField
from common.utils import get_logger
from .base import BaseAutomationSerializer
logger = get_logger(__file__)
__all__ = ['BackupAccountSerializer', 'BackupAccountExecutionSerializer']
__all__ = ['BackupAccountSerializer']
class BackupAccountSerializer(BaseAutomationSerializer):
@ -41,14 +40,6 @@ class BackupAccountSerializer(BaseAutomationSerializer):
'types': {'label': _('Asset type')}
}
class BackupAccountExecutionSerializer(serializers.ModelSerializer):
trigger = LabeledChoiceField(choices=Trigger.choices, label=_("Trigger mode"), read_only=True)
class Meta:
model = BackupAccountAutomation
read_only_fields = [
'id', 'date_start', 'timedelta', 'snapshot',
'trigger', 'reason', 'is_success', 'org_id'
]
fields = read_only_fields + ['plan']
@property
def model_type(self):
return AutomationTypes.backup_account

7
apps/accounts/serializers/automations/base.py
View File

@ -26,15 +26,14 @@ class BaseAutomationSerializer(PeriodTaskSerializerMixin, BulkOrgResourceModelSe
class Meta:
read_only_fields = [
'date_created', 'date_updated', 'created_by',
'periodic_display', 'executed_amount'
'periodic_display', 'executed_amount', 'type'
]
fields = read_only_fields + [
'id', 'name', 'is_periodic', 'interval', 'crontab', 'comment',
'type', 'accounts', 'nodes', 'assets', 'is_active',
'accounts', 'nodes', 'assets', 'is_active',
]
extra_kwargs = {
'name': {'required': True},
'type': {'read_only': True},
'executed_amount': {'label': _('Executions')},
}
@ -58,7 +57,7 @@ class AutomationExecutionSerializer(serializers.ModelSerializer):
class Meta:
model = AutomationExecution
read_only_fields = [
'trigger', 'date_start', 'date_finished', 'snapshot', 'status'
'trigger', 'date_start', 'date_finished', 'snapshot', 'status', 'duration'
]
fields = ['id', 'automation', 'type'] + read_only_fields

78
apps/accounts/templates/accounts/backup_account_report.html Normal file
View File

@ -0,0 +1,78 @@
{% load i18n %}
<div class='summary'>
<p>{% trans 'The following is a summary of account backup tasks, please review and handle them' %}</p>
<table>
<caption></caption>
<thead>
<tr>
<th colspan='2'>任务汇总:</th>
</tr>
</thead>
<tbody>
<tr>
<td>{% trans 'Task name' %}:</td>
<td>{{ execution.automation.name }} </td>
</tr>
<tr>
<td>{% trans 'Date start' %}:</td>
<td>{{ execution.date_start | date:"Y/m/d H:i:s" }}</td>
</tr>
<tr>
<td>{% trans 'Date end' %}:</td>
<td>{{ execution.date_finished | date:"Y/m/d H:i:s" }}</td>
</tr>
<tr>
<td>{% trans 'Time using' %}:</td>
<td>{{ execution.duration }}s</td>
</tr>
<tr>
<td>{% trans 'Account count' %}:</td>
<td>{{ summary.total_accounts }}</td>
</tr>
<tr>
<td>{% trans 'Type count' %}:</td>
<td>{{ summary.total_types }}</td>
</tr>
</tbody>
</table>
</div>
<style>
table {
width: 100%;
border-collapse: collapse;
max-width: 100%;
text-align: left;
margin-top: 10px;
padding: 20px;
}
th {
background: #f2f2f2;
font-size: 14px;
padding: 5px;
border: 1px solid #ddd;
}
tr :first-child {
width: 30%;
}
td {
border: 1px solid #ddd;
padding: 5px;
font-size: 12px;
}
.result {
margin-top: 20px;
}
.result tr :first-child {
width: 10%;
}
</style>

2
apps/accounts/urls.py
View File

@ -15,7 +15,7 @@ router.register(r'gathered-accounts', api.GatheredAccountViewSet, 'gathered-acco
router.register(r'account-secrets', api.AccountSecretsViewSet, 'account-secret')
router.register(r'account-templates', api.AccountTemplateViewSet, 'account-template')
router.register(r'account-template-secrets', api.AccountTemplateSecretsViewSet, 'account-template-secret')
router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup')
router.register(r'account-backup-plans', api.BackupAccountViewSet, 'account-backup')
router.register(r'account-backup-plan-executions', api.BackupAccountExecutionViewSet, 'account-backup-execution')
router.register(r'change-secret-automations', api.ChangeSecretAutomationViewSet, 'change-secret-automation')
router.register(r'change-secret-executions', api.ChangSecretExecutionViewSet, 'change-secret-execution')

2
apps/assets/automations/base/manager.py
View File

@ -98,7 +98,7 @@ class BaseManager:
self.summary = defaultdict(int)
self.result = defaultdict(list)
self.duration = 0
self.status = 'success'
self.status = Status.success
def get_assets_group_by_platform(self):
return self.execution.all_assets_group_by_platform()

12
apps/assets/serializers/automations/base.py
View File

@ -42,17 +42,17 @@ class AutomationExecutionSerializer(serializers.ModelSerializer):
class Meta:
model = AutomationExecution
read_only_fields = [
'trigger', 'date_start', 'date_finished', 'snapshot', 'status'
'trigger', 'date_start', 'date_finished', 'snapshot', 'status', 'duration'
]
fields = ['id', 'automation'] + read_only_fields
@staticmethod
def get_status(obj):
if obj.status == 'success':
return _("Success")
elif obj.status == 'pending':
return _("Pending")
return obj.status
from common.const import Status
status = Status._member_map_.get(obj.status)
if status is None:
return obj.status
return status.label
@staticmethod
def get_snapshot(obj):