perf: 改密记录可查看密文 (#12821)

* perf: 改密记录可查看密文

* perf: 自动化任务错误处理

* feat: 改密记录可批量重试 新增更多过滤选项

* perf: 改密任务失败添加消息通知

---------

Co-authored-by: feng <1304903146@qq.com>

apps/accounts/api/automations/change_secret.py

@@ -6,9 +6,12 @@ from rest_framework.response import Response
 
 from accounts import serializers
 from accounts.const import AutomationTypes
+from accounts.filters import ChangeSecretRecordFilterSet
 from accounts.models import ChangeSecretAutomation, ChangeSecretRecord
 from accounts.tasks import execute_automation_record_task
+from authentication.permissions import UserConfirmation, ConfirmType
 from orgs.mixins.api import OrgBulkModelViewSet, OrgGenericViewSet
+from rbac.permissions import RBACPermission
 from .base import (
     AutomationAssetsListApi, AutomationRemoveAssetApi, AutomationAddAssetApi,
     AutomationNodeAddRemoveApi, AutomationExecutionViewSet
@@ -30,29 +33,48 @@ class ChangeSecretAutomationViewSet(OrgBulkModelViewSet):
 
 
 class ChangeSecretRecordViewSet(mixins.ListModelMixin, OrgGenericViewSet):
-    serializer_class = serializers.ChangeSecretRecordSerializer
-    filterset_fields = ('asset_id', 'execution_id')
+    filterset_class = ChangeSecretRecordFilterSet
     search_fields = ('asset__address',)
     tp = AutomationTypes.change_secret
+    serializer_classes = {
+        'default': serializers.ChangeSecretRecordSerializer,
+        'secret': serializers.ChangeSecretRecordViewSecretSerializer,
+    }
     rbac_perms = {
         'execute': 'accounts.add_changesecretexecution',
+        'secret': 'accounts.view_changesecretrecord',
     }
 
+    def get_permissions(self):
+        if self.action == 'secret':
+            self.permission_classes = [
+                RBACPermission,
+                UserConfirmation.require(ConfirmType.MFA)
+            ]
+        return super().get_permissions()
+
     def get_queryset(self):
         return ChangeSecretRecord.objects.all()
 
     @action(methods=['post'], detail=False, url_path='execute')
     def execute(self, request, *args, **kwargs):
-        record_id = request.data.get('record_id')
-        record = self.get_queryset().filter(pk=record_id)
-        if not record:
+        record_ids = request.data.get('record_ids')
+        records = self.get_queryset().filter(id__in=record_ids)
+        execution_count = records.values_list('execution_id', flat=True).distinct().count()
+        if execution_count != 1:
             return Response(
-                {'detail': 'record not found'},
-                status=status.HTTP_404_NOT_FOUND
+                {'detail': 'Only one execution is allowed to execute'},
+                status=status.HTTP_400_BAD_REQUEST
             )
-        task = execute_automation_record_task.delay(record_id, self.tp)
+        task = execute_automation_record_task.delay(record_ids, self.tp)
         return Response({'task': task.id}, status=status.HTTP_200_OK)
 
+    @action(methods=['get'], detail=True, url_path='secret')
+    def secret(self, request, *args, **kwargs):
+        instance = self.get_object()
+        serializer = self.get_serializer(instance)
+        return Response(serializer.data)
+
 
 class ChangSecretExecutionViewSet(AutomationExecutionViewSet):
     rbac_perms = (

apps/accounts/automations/backup_account/handlers.py

@@ -6,7 +6,7 @@ from django.conf import settings
 from rest_framework import serializers
 from xlsxwriter import Workbook
 
-from accounts.const.automation import AccountBackupType
+from accounts.const import AccountBackupType
 from accounts.models.automations.backup_account import AccountBackupAutomation
 from accounts.notifications import AccountBackupExecutionTaskMsg, AccountBackupByObjStorageExecutionTaskMsg
 from accounts.serializers import AccountSecretSerializer

apps/accounts/automations/change_secret/manager.py

@@ -7,9 +7,9 @@ from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from xlsxwriter import Workbook
 
-from accounts.const import AutomationTypes, SecretType, SSHKeyStrategy, SecretStrategy
+from accounts.const import AutomationTypes, SecretType, SSHKeyStrategy, SecretStrategy, ChangeSecretRecordStatusChoice
 from accounts.models import ChangeSecretRecord
-from accounts.notifications import ChangeSecretExecutionTaskMsg
+from accounts.notifications import ChangeSecretExecutionTaskMsg, ChangeSecretFailedMsg
 from accounts.serializers import ChangeSecretRecordBackUpSerializer
 from assets.const import HostTypes
 from common.utils import get_logger
@@ -27,7 +27,7 @@ class ChangeSecretManager(AccountBasePlaybookManager):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.record_id = self.execution.snapshot.get('record_id')
+        self.record_map = self.execution.snapshot.get('record_map', {})
         self.secret_type = self.execution.snapshot.get('secret_type')
         self.secret_strategy = self.execution.snapshot.get(
             'secret_strategy', SecretStrategy.custom
@@ -123,14 +123,20 @@ class ChangeSecretManager(AccountBasePlaybookManager):
                 print(f'new_secret is None, account: {account}')
                 continue
 
-            if self.record_id is None:
+            asset_account_id = f'{asset.id}-{account.id}'
+            if asset_account_id not in self.record_map:
                 recorder = ChangeSecretRecord(
                     asset=asset, account=account, execution=self.execution,
                     old_secret=account.secret, new_secret=new_secret,
                 )
                 records.append(recorder)
             else:
-                recorder = ChangeSecretRecord.objects.get(id=self.record_id)
+                record_id = self.record_map[asset_account_id]
+                try:
+                    recorder = ChangeSecretRecord.objects.get(id=record_id)
+                except ChangeSecretRecord.DoesNotExist:
+                    print(f"Record {record_id} not found")
+                    continue
 
             self.name_recorder_mapper[h['name']] = recorder
 
@@ -158,25 +164,43 @@ class ChangeSecretManager(AccountBasePlaybookManager):
         recorder = self.name_recorder_mapper.get(host)
         if not recorder:
             return
-        recorder.status = 'success'
+        recorder.status = ChangeSecretRecordStatusChoice.success.value
         recorder.date_finished = timezone.now()
-        recorder.save()
+
         account = recorder.account
         if not account:
             print("Account not found, deleted ?")
             return
         account.secret = recorder.new_secret
         account.date_updated = timezone.now()
-        account.save(update_fields=['secret', 'date_updated'])
+
+        max_retries = 3
+        retry_count = 0
+
+        while retry_count < max_retries:
+            try:
+                recorder.save()
+                account.save(update_fields=['secret', 'date_updated'])
+                break
+            except Exception as e:
+                retry_count += 1
+                if retry_count == max_retries:
+                    self.on_host_error(host, str(e), result)
+                else:
+                    print(f'retry {retry_count} times for {host} recorder save error: {e}')
+                    time.sleep(1)
 
     def on_host_error(self, host, error, result):
         recorder = self.name_recorder_mapper.get(host)
         if not recorder:
             return
-        recorder.status = 'failed'
+        recorder.status = ChangeSecretRecordStatusChoice.failed.value
         recorder.date_finished = timezone.now()
         recorder.error = error
-        recorder.save()
+        try:
+            recorder.save()
+        except Exception as e:
+            print(f"\033[31m Save {host} recorder error: {e} \033[0m\n")
 
     def on_runner_failed(self, runner, e):
         logger.error("Account error: ", e)
@@ -192,7 +216,7 @@ class ChangeSecretManager(AccountBasePlaybookManager):
     def get_summary(recorders):
         total, succeed, failed = 0, 0, 0
         for recorder in recorders:
-            if recorder.status == 'success':
+            if recorder.status == ChangeSecretRecordStatusChoice.success.value:
                 succeed += 1
             else:
                 failed += 1
@@ -209,9 +233,25 @@ class ChangeSecretManager(AccountBasePlaybookManager):
         summary = self.get_summary(recorders)
         print(summary, end='')
 
-        if self.record_id:
+        if self.record_map:
             return
 
+        failed_recorders = [
+            r for r in recorders
+            if r.status == ChangeSecretRecordStatusChoice.failed.value
+        ]
+        super_users = User.get_super_admins()
+
+        if failed_recorders and super_users:
+            name = self.execution.snapshot.get('name')
+            execution_id = str(self.execution.id)
+            _ids = [r.id for r in failed_recorders]
+            asset_account_errors = ChangeSecretRecord.objects.filter(
+                id__in=_ids).values_list('asset__name', 'account__username', 'error')
+
+            for user in super_users:
+                ChangeSecretFailedMsg(name, execution_id, user, asset_account_errors).publish()
+
         self.send_recorder_mail(recorders, summary)
 
     def send_recorder_mail(self, recorders, summary):

apps/accounts/automations/gather_accounts/manager.py

@@ -58,7 +58,7 @@ class GatherAccountsManager(AccountBasePlaybookManager):
             result = self.filter_success_result(asset.type, info)
             self.collect_asset_account_info(asset, result)
         else:
-            logger.error(f'Not found {host} info')
+            print(f'\033[31m Not found {host} info \033[0m\n')
 
     def update_or_create_accounts(self):
         for asset, data in self.asset_account_info.items():

apps/accounts/automations/remove_account/manager.py

@@ -60,8 +60,11 @@ class RemoveAccountManager(AccountBasePlaybookManager):
         if not tuple_asset_gather_account:
             return
         asset, gather_account = tuple_asset_gather_account
-        Account.objects.filter(
-            asset_id=asset.id,
-            username=gather_account.username
-        ).delete()
-        gather_account.delete()
+        try:
+            Account.objects.filter(
+                asset_id=asset.id,
+                username=gather_account.username
+            ).delete()
+            gather_account.delete()
+        except Exception as e:
+            print(f'\033[31m Delete account {gather_account.username} failed: {e} \033[0m\n')

apps/accounts/automations/verify_account/manager.py

@@ -76,8 +76,14 @@ class VerifyAccountManager(AccountBasePlaybookManager):
 
     def on_host_success(self, host, result):
         account = self.host_account_mapper.get(host)
-        account.set_connectivity(Connectivity.OK)
+        try:
+            account.set_connectivity(Connectivity.OK)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} connectivity failed: {e} \033[0m\n')
 
     def on_host_error(self, host, error, result):
         account = self.host_account_mapper.get(host)
-        account.set_connectivity(Connectivity.ERR)
+        try:
+            account.set_connectivity(Connectivity.ERR)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} connectivity failed: {e} \033[0m\n')

apps/accounts/const/automation.py

@@ -16,7 +16,7 @@ DEFAULT_PASSWORD_RULES = {
 __all__ = [
     'AutomationTypes', 'SecretStrategy', 'SSHKeyStrategy', 'Connectivity',
     'DEFAULT_PASSWORD_LENGTH', 'DEFAULT_PASSWORD_RULES', 'TriggerChoice',
-    'PushAccountActionChoice', 'AccountBackupType'
+    'PushAccountActionChoice', 'AccountBackupType', 'ChangeSecretRecordStatusChoice',
 ]
 
 
@@ -103,3 +103,9 @@ class AccountBackupType(models.TextChoices):
     email = 'email', _('Email')
     # 目前只支持sftp方式
     object_storage = 'object_storage', _('SFTP')
+
+
+class ChangeSecretRecordStatusChoice(models.TextChoices):
+    failed = 'failed', _('Failed')
+    success = 'success', _('Success')
+    pending = 'pending', _('Pending')

apps/accounts/filters.py

@@ -5,7 +5,7 @@ from django_filters import rest_framework as drf_filters
 
 from assets.models import Node
 from common.drf.filters import BaseFilterSet
-from .models import Account, GatheredAccount
+from .models import Account, GatheredAccount, ChangeSecretRecord
 
 
 class AccountFilterSet(BaseFilterSet):
@@ -61,3 +61,12 @@ class GatheredAccountFilterSet(BaseFilterSet):
     class Meta:
         model = GatheredAccount
         fields = ['id', 'username']
+
+
+class ChangeSecretRecordFilterSet(BaseFilterSet):
+    asset_name = drf_filters.CharFilter(field_name='asset__name', lookup_expr='icontains')
+    account_username = drf_filters.CharFilter(field_name='account__username', lookup_expr='icontains')
+
+    class Meta:
+        model = ChangeSecretRecord
+        fields = ['id', 'status', 'asset_id', 'execution_id']

apps/accounts/models/automations/backup_account.py

@@ -8,7 +8,7 @@ from django.db import models
 from django.db.models import F
 from django.utils.translation import gettext_lazy as _
 
-from accounts.const.automation import AccountBackupType
+from accounts.const import AccountBackupType
 from common.const.choices import Trigger
 from common.db import fields
 from common.db.encoder import ModelJSONFieldEncoder

apps/accounts/models/automations/change_secret.py

@@ -2,7 +2,7 @@ from django.db import models
 from django.utils.translation import gettext_lazy as _
 
 from accounts.const import (
-    AutomationTypes
+    AutomationTypes, ChangeSecretRecordStatusChoice
 )
 from common.db import fields
 from common.db.models import JMSBaseModel
@@ -40,7 +40,10 @@ class ChangeSecretRecord(JMSBaseModel):
     new_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('New secret'))
     date_started = models.DateTimeField(blank=True, null=True, verbose_name=_('Date started'))
     date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('Date finished'))
-    status = models.CharField(max_length=16, default='pending', verbose_name=_('Status'))
+    status = models.CharField(
+        max_length=16, verbose_name=_('Status'),
+        default=ChangeSecretRecordStatusChoice.pending.value
+    )
     error = models.TextField(blank=True, null=True, verbose_name=_('Error'))
 
     class Meta:

apps/accounts/notifications.py

@@ -1,6 +1,7 @@
 from django.template.loader import render_to_string
 from django.utils.translation import gettext_lazy as _
 
+from accounts.models import ChangeSecretRecord
 from common.tasks import send_mail_attachment_async, upload_backup_to_obj_storage
 from notifications.notifications import UserMessage
 from terminal.models.component.storage import ReplayStorage
@@ -98,3 +99,33 @@ class GatherAccountChangeMsg(UserMessage):
     def gen_test_msg(cls):
         user = User.objects.first()
         return cls(user, {})
+
+
+class ChangeSecretFailedMsg(UserMessage):
+    subject = _('Change secret or push account failed information')
+
+    def __init__(self, name, execution_id, user, asset_account_errors: list):
+        self.name = name
+        self.execution_id = execution_id
+        self.asset_account_errors = asset_account_errors
+        super().__init__(user)
+
+    def get_html_msg(self) -> dict:
+        context = {
+            'name': self.name,
+            'recipient': self.user,
+            'execution_id': self.execution_id,
+            'asset_account_errors': self.asset_account_errors
+        }
+        message = render_to_string('accounts/change_secret_failed_info.html', context)
+
+        return {
+            'subject': str(self.subject),
+            'message': message
+        }
+
+    @classmethod
+    def gen_test_msg(cls):
+        user = User.objects.first()
+        record = ChangeSecretRecord.objects.first()
+        return cls(user, [record])

apps/accounts/serializers/automations/change_secret.py

@@ -4,7 +4,8 @@ from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.const import (
-    AutomationTypes, SecretType, SecretStrategy, SSHKeyStrategy
+    AutomationTypes, SecretType, SecretStrategy,
+    SSHKeyStrategy, ChangeSecretRecordStatusChoice
 )
 from accounts.models import (
     Account, ChangeSecretAutomation,
@@ -21,6 +22,7 @@ logger = get_logger(__file__)
 __all__ = [
     'ChangeSecretAutomationSerializer',
     'ChangeSecretRecordSerializer',
+    'ChangeSecretRecordViewSecretSerializer',
     'ChangeSecretRecordBackUpSerializer',
     'ChangeSecretUpdateAssetSerializer',
     'ChangeSecretUpdateNodeSerializer',
@@ -104,7 +106,10 @@ class ChangeSecretAutomationSerializer(AuthValidateMixin, BaseAutomationSerializ
 class ChangeSecretRecordSerializer(serializers.ModelSerializer):
     is_success = serializers.SerializerMethodField(label=_('Is success'))
     asset = ObjectRelatedField(queryset=Asset.objects, label=_('Asset'))
-    account = ObjectRelatedField(queryset=Account.objects, label=_('Account'))
+    account = ObjectRelatedField(
+        queryset=Account.objects, label=_('Account'),
+        attrs=("id", "name", "username")
+    )
     execution = ObjectRelatedField(
         queryset=AutomationExecution.objects, label=_('Automation task execution')
     )
@@ -119,7 +124,16 @@ class ChangeSecretRecordSerializer(serializers.ModelSerializer):
 
     @staticmethod
     def get_is_success(obj):
-        return obj.status == 'success'
+        return obj.status == ChangeSecretRecordStatusChoice.success.value
+
+
+class ChangeSecretRecordViewSecretSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ChangeSecretRecord
+        fields = [
+            'id', 'old_secret', 'new_secret',
+        ]
+        read_only_fields = fields
 
 
 class ChangeSecretRecordBackUpSerializer(serializers.ModelSerializer):
@@ -145,7 +159,7 @@ class ChangeSecretRecordBackUpSerializer(serializers.ModelSerializer):
 
     @staticmethod
     def get_is_success(obj):
-        if obj.status == 'success':
+        if obj.status == ChangeSecretRecordStatusChoice.success.value:
             return _("Success")
         return _("Failed")
 

apps/accounts/tasks/automation.py

@@ -36,14 +36,14 @@ def execute_account_automation_task(pid, trigger, tp):
         instance.execute(trigger)
 
 
-def record_task_activity_callback(self, record_id, *args, **kwargs):
+def record_task_activity_callback(self, record_ids, *args, **kwargs):
     from accounts.models import ChangeSecretRecord
     with tmp_to_root_org():
-        record = get_object_or_none(ChangeSecretRecord, id=record_id)
-    if not record:
+        records = ChangeSecretRecord.objects.filter(id__in=record_ids)
+    if not records:
         return
-    resource_ids = [record.id]
-    org_id = record.execution.org_id
+    resource_ids = [str(i.id) for i in records]
+    org_id = records[0].execution.org_id
     return resource_ids, org_id
 
 
@@ -51,22 +51,26 @@ def record_task_activity_callback(self, record_id, *args, **kwargs):
     queue='ansible', verbose_name=_('Execute automation record'),
     activity_callback=record_task_activity_callback
 )
-def execute_automation_record_task(record_id, tp):
+def execute_automation_record_task(record_ids, tp):
     from accounts.models import ChangeSecretRecord
+    task_name = gettext_noop('Execute automation record')
+
     with tmp_to_root_org():
-        instance = get_object_or_none(ChangeSecretRecord, pk=record_id)
-    if not instance:
-        logger.error("No automation record found: {}".format(record_id))
+        records = ChangeSecretRecord.objects.filter(id__in=record_ids)
+
+    if not records:
+        logger.error('No automation record found: {}'.format(record_ids))
         return
 
-    task_name = gettext_noop('Execute automation record')
+    record = records[0]
+    record_map = {f'{record.asset_id}-{record.account_id}': str(record.id) for record in records}
     task_snapshot = {
-        'secret': instance.new_secret,
-        'secret_type': instance.execution.snapshot.get('secret_type'),
-        'accounts': [str(instance.account_id)],
-        'assets': [str(instance.asset_id)],
         'params': {},
-        'record_id': record_id,
+        'record_map': record_map,
+        'secret': record.new_secret,
+        'secret_type': record.execution.snapshot.get('secret_type'),
+        'assets': [str(instance.asset_id) for instance in records],
+        'accounts': [str(instance.account_id) for instance in records],
     }
-    with tmp_to_org(instance.execution.org_id):
+    with tmp_to_org(record.execution.org_id):
         quickstart_automation_by_snapshot(task_name, tp, task_snapshot)

apps/accounts/templates/accounts/asset_account_change_info.html

@@ -1,10 +1,10 @@
 {% load i18n %}
 
-
+<h3>{% trans 'Gather account change information' %}</h3>
 <table style="width: 100%; border-collapse: collapse; max-width: 100%; text-align: left; margin-top: 20px;">
     <caption></caption>
     <tr style="background-color: #f2f2f2;">
-        <th style="border: 1px solid #ddd; padding: 10px; font-weight: bold;">{% trans 'Asset' %}</th>
+        <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Asset' %}</th>
         <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Added account' %}</th>
         <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Deleted account' %}</th>
     </tr>

apps/accounts/templates/accounts/change_secret_failed_info.html

@@ -0,0 +1,36 @@
+{% load i18n %}
+
+<h3>{% trans 'Task name' %}: {{ name }}</h3>
+<h3>{% trans 'Task execution id' %}: {{ execution_id }}</h3>
+<p>{% trans 'Respectful' %} {{ recipient }}</p>
+<p>{% trans 'Hello! The following is the failure of changing the password of your assets or pushing the account. Please check and handle it in time.' %}</p>
+<table style="width: 100%; border-collapse: collapse; max-width: 100%; text-align: left; margin-top: 20px;">
+    <caption></caption>
+    <thead>
+    <tr style="background-color: #f2f2f2;">
+        <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Asset' %}</th>
+        <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Account' %}</th>
+        <th style="border: 1px solid #ddd; padding: 10px;">{% trans 'Error' %}</th>
+    </tr>
+    </thead>
+    <tbody>
+    {% for asset_name, account_username, error in asset_account_errors %}
+        <tr>
+            <td style="border: 1px solid #ddd; padding: 10px;">{{ asset_name }}</td>
+            <td style="border: 1px solid #ddd; padding: 10px;">{{ account_username }}</td>
+            <td style="border: 1px solid #ddd; padding: 10px;">
+                <div style="
+                max-width: 90%;
+                white-space: nowrap;
+                overflow: hidden;
+                text-overflow: ellipsis;
+                display: block;"
+                     title="{{ error }}"
+                >
+                    {{ error }}
+                </div>
+            </td>
+        </tr>
+    {% endfor %}
+    </tbody>
+</table>

apps/assets/automations/ping/manager.py

@@ -25,14 +25,22 @@ class PingManager(BasePlaybookManager):
 
     def on_host_success(self, host, result):
         asset, account = self.host_asset_and_account_mapper.get(host)
-        asset.set_connectivity(Connectivity.OK)
-        if not account:
-            return
-        account.set_connectivity(Connectivity.OK)
+        try:
+            asset.set_connectivity(Connectivity.OK)
+            if not account:
+                return
+            account.set_connectivity(Connectivity.OK)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} or '
+                  f'update asset {asset.name} connectivity failed: {e} \033[0m\n')
 
     def on_host_error(self, host, error, result):
         asset, account = self.host_asset_and_account_mapper.get(host)
-        asset.set_connectivity(Connectivity.ERR)
-        if not account:
-            return
-        account.set_connectivity(Connectivity.ERR)
+        try:
+            asset.set_connectivity(Connectivity.ERR)
+            if not account:
+                return
+            account.set_connectivity(Connectivity.ERR)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} or '
+                  f'update asset {asset.name} connectivity failed: {e} \033[0m\n')

apps/assets/automations/ping_gateway/manager.py

@@ -92,18 +92,26 @@ class PingGatewayManager:
     @staticmethod
     def on_host_success(gateway, account):
         print('\033[32m {} -> {}\033[0m\n'.format(gateway, account))
-        gateway.set_connectivity(Connectivity.OK)
-        if not account:
-            return
-        account.set_connectivity(Connectivity.OK)
+        try:
+            gateway.set_connectivity(Connectivity.OK)
+            if not account:
+                return
+            account.set_connectivity(Connectivity.OK)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} or '
+                  f'update asset {gateway.name} connectivity failed: {e} \033[0m\n')
 
     @staticmethod
     def on_host_error(gateway, account, error):
         print('\033[31m {} -> {} 原因: {} \033[0m\n'.format(gateway, account, error))
-        gateway.set_connectivity(Connectivity.ERR)
-        if not account:
-            return
-        account.set_connectivity(Connectivity.ERR)
+        try:
+            gateway.set_connectivity(Connectivity.ERR)
+            if not account:
+                return
+            account.set_connectivity(Connectivity.ERR)
+        except Exception as e:
+            print(f'\033[31m Update account {account.name} or '
+                  f'update asset {gateway.name} connectivity failed: {e} \033[0m\n')
 
     @staticmethod
     def before_runner_start():