From 55c7e06185b0e2aee2d023819dde3e0a2ab85c69 Mon Sep 17 00:00:00 2001 From: Michael Bai Date: Tue, 24 Mar 2020 01:37:09 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[Update]=20=E4=BF=AE=E6=94=B9=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=89=A7=E8=A1=8C=E6=97=A5=E6=9C=9F=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/models/adhoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index c29493fb2..013fdc628 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -278,7 +278,7 @@ class AdHocExecution(OrgModelMixin): raw = '' try: - date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + date_start_s = timezone.now().now().strftime('%Y-%m-%d %H:%M:%S') print(_("{} Start task: {}").format(date_start_s, self.task.name)) raw, summary = self.start_runner() except Exception as e: @@ -286,7 +286,7 @@ class AdHocExecution(OrgModelMixin): raw = {"dark": {"all": str(e)}, "contacted": []} finally: self.clean_up(summary, time_start) - date_end = timezone.now() + date_end = timezone.now().now() date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S') print(_("{} Task finish").format(date_end_s)) print('.\n\n.') From 4503df910d2081ed71215acf55b8f935199f0db6 Mon Sep 17 00:00:00 2001 From: Bai Date: Tue, 24 Mar 2020 18:40:20 +0800 Subject: [PATCH 2/3] =?UTF-8?q?[Update]=20=E4=BC=98=E5=8C=96=E5=88=9B?= =?UTF-8?q?=E5=BB=BAAuthBook=E9=80=BB=E8=BE=91=EF=BC=9B=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E9=94=81=E6=9C=BA=E5=88=B6(=E5=9F=BA=E4=BA=8Eredis-lock)?= =?UTF-8?q?=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/backends/manager.py | 4 +-- apps/assets/models/authbook.py | 51 ++++++++++++--------------- apps/assets/serializers/asset_user.py | 1 - apps/assets/signals_handler.py | 7 ---- apps/jumpserver/settings/base.py | 3 +- requirements/requirements.txt | 2 ++ 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/apps/assets/backends/manager.py b/apps/assets/backends/manager.py index d686cb922..5d1966688 100644 --- a/apps/assets/backends/manager.py +++ b/apps/assets/backends/manager.py @@ -154,8 +154,8 @@ class AssetUserManager: @staticmethod def create(**kwargs): - authbook = AuthBook(**kwargs) - authbook.save() + # 使用create方法创建AuthBook对象，解决并发创建问题（添加锁机制） + authbook = AuthBook.objects.create(**kwargs) return authbook def __getattr__(self, item): diff --git a/apps/assets/models/authbook.py b/apps/assets/models/authbook.py index 0a0154100..2c96f9bdb 100644 --- a/apps/assets/models/authbook.py +++ b/apps/assets/models/authbook.py @@ -2,6 +2,8 @@ # from django.db import models +from django.db.models import Max +from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from orgs.mixins.models import OrgManager @@ -11,12 +13,30 @@ __all__ = ['AuthBook'] class AuthBookQuerySet(models.QuerySet): - def latest_version(self): - return self.filter(is_latest=True) + def delete(self): + raise PermissionError("Bulk delete authbook deny") class AuthBookManager(OrgManager): - pass + + def get_max_version(self, username, asset): + version_max = self.filter(username=username, asset=asset)\ + .aggregate(Max('version')) + version_max = version_max['version__max'] or 0 + return version_max + + def create(self, **kwargs): + username = kwargs['username'] + asset = kwargs['asset'] + key_lock = 'KEY_LOCK_CREATE_AUTH_BOOK_{}_{}'.format(username, asset.id) + with cache.lock(key_lock, expire=60): + self.filter(username=username, asset=asset, is_latest=True)\ + .update(is_latest=False) + max_version = self.get_max_version(username, asset) + kwargs['version'] = max_version + 1 + kwargs['is_latest'] = True + obj = super().create(**kwargs) + return obj class AuthBook(BaseUser): @@ -33,31 +53,6 @@ class AuthBook(BaseUser): class Meta: verbose_name = _('AuthBook') - def set_to_latest(self): - self.remove_pre_latest() - self.is_latest = True - self.save() - - def get_pre_latest(self): - pre_obj = self.__class__.objects.filter( - username=self.username, asset=self.asset - ).latest_version().first() - return pre_obj - - def remove_pre_latest(self): - pre_obj = self.get_pre_latest() - if pre_obj: - pre_obj.is_latest = False - pre_obj.save() - - def set_version(self): - pre_obj = self.get_pre_latest() - if pre_obj: - self.version = pre_obj.version + 1 - else: - self.version = 1 - self.save() - def get_related_assets(self): return [self.asset] diff --git a/apps/assets/serializers/asset_user.py b/apps/assets/serializers/asset_user.py index 896ad9bef..1c598cbf5 100644 --- a/apps/assets/serializers/asset_user.py +++ b/apps/assets/serializers/asset_user.py @@ -37,7 +37,6 @@ class AssetUserWriteSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializ if not validated_data.get("name") and validated_data.get("username"): validated_data["name"] = validated_data["username"] instance = AssetUserManager.create(**validated_data) - instance.set_to_latest() return instance diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 3bf185bc4..ce906dcd2 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -15,7 +15,6 @@ from .utils import TreeService from .tasks import ( update_assets_hardware_info_util, test_asset_connectivity_util, - push_system_user_to_assets, push_system_user_to_assets_manual, push_system_user_to_assets, add_nodes_assets_to_system_users @@ -235,9 +234,3 @@ def on_node_update_or_created(sender, **kwargs): Node.refresh_nodes() with tmp_to_root_org(): Node.refresh_nodes() - - -@receiver(post_save, sender=AuthBook) -def on_authbook_created(sender, instance=None, created=True, **kwargs): - if created and instance: - instance.set_version() diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index 22ebd1775..4a2f2ca88 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -232,7 +232,8 @@ FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o755 # Cache use redis CACHES = { 'default': { - 'BACKEND': 'redis_cache.RedisCache', + # 'BACKEND': 'redis_cache.RedisCache', + 'BACKEND': 'redis_lock.django_cache.RedisCache', 'LOCATION': 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % { 'password': CONFIG.REDIS_PASSWORD, 'host': CONFIG.REDIS_HOST, diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c9867b4e7..63bc4f8d4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -96,3 +96,5 @@ django-cas-ng==4.0.1 python-cas==1.5.0 ipython huaweicloud-sdk-python==1.0.21 +django-redis==4.11.0 +python-redis-lock==3.5.0 From 4583beaec0b362156471a658ef3b68246d0ba36a Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 26 Mar 2020 13:11:38 +0800 Subject: [PATCH 3/3] =?UTF-8?q?[Update]=20=E5=88=9B=E5=BB=BAAuthBook?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=E7=9A=84=E6=96=B9=E6=B3=95=E4=BB=8EModelMana?= =?UTF-8?q?ger=E4=B8=AD=E7=A7=BB=E5=8A=A8=E5=88=B0Model=E4=B8=AD=EF=BC=8C?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E5=8E=9F=E7=94=9F=E7=9A=84=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?create=E6=96=B9=E6=B3=95=EF=BC=9B=E5=90=8C=E6=97=B6=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BA=8B=E5=8A=A1=E5=A4=84=E7=90=86=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/backends/manager.py | 2 +- apps/assets/models/authbook.py | 52 ++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/apps/assets/backends/manager.py b/apps/assets/backends/manager.py index 5d1966688..ee6650ed5 100644 --- a/apps/assets/backends/manager.py +++ b/apps/assets/backends/manager.py @@ -155,7 +155,7 @@ class AssetUserManager: @staticmethod def create(**kwargs): # 使用create方法创建AuthBook对象，解决并发创建问题（添加锁机制） - authbook = AuthBook.objects.create(**kwargs) + authbook = AuthBook.create(**kwargs) return authbook def __getattr__(self, item): diff --git a/apps/assets/models/authbook.py b/apps/assets/models/authbook.py index 2c96f9bdb..2c8c0c051 100644 --- a/apps/assets/models/authbook.py +++ b/apps/assets/models/authbook.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -from django.db import models +from django.db import models, transaction from django.db.models import Max from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ @@ -18,25 +18,7 @@ class AuthBookQuerySet(models.QuerySet): class AuthBookManager(OrgManager): - - def get_max_version(self, username, asset): - version_max = self.filter(username=username, asset=asset)\ - .aggregate(Max('version')) - version_max = version_max['version__max'] or 0 - return version_max - - def create(self, **kwargs): - username = kwargs['username'] - asset = kwargs['asset'] - key_lock = 'KEY_LOCK_CREATE_AUTH_BOOK_{}_{}'.format(username, asset.id) - with cache.lock(key_lock, expire=60): - self.filter(username=username, asset=asset, is_latest=True)\ - .update(is_latest=False) - max_version = self.get_max_version(username, asset) - kwargs['version'] = max_version + 1 - kwargs['is_latest'] = True - obj = super().create(**kwargs) - return obj + pass class AuthBook(BaseUser): @@ -59,6 +41,36 @@ class AuthBook(BaseUser): def generate_id_with_asset(self, asset): return self.id + @classmethod + def get_max_version(cls, username, asset): + version_max = cls.objects.filter(username=username, asset=asset) \ + .aggregate(Max('version')) + version_max = version_max['version__max'] or 0 + return version_max + + @classmethod + def create(cls, **kwargs): + """ + 使用并发锁机制创建AuthBook对象, (主要针对并发创建 username, asset 相同的对象时) + 并更新其他对象的 is_latest=False (其他对象: 与当前对象的 username, asset 相同) + 同时设置自己的 is_latest=True, version=max_version + 1 + """ + username = kwargs['username'] + asset = kwargs['asset'] + key_lock = 'KEY_LOCK_CREATE_AUTH_BOOK_{}_{}'.format(username, asset.id) + with cache.lock(key_lock): + with transaction.atomic(): + cls.objects.filter( + username=username, asset=asset, is_latest=True + ).update(is_latest=False) + max_version = cls.get_max_version(username, asset) + kwargs.update({ + 'version': max_version + 1, + 'is_latest': True + }) + obj = cls.objects.create(**kwargs) + return obj + @property def connectivity(self): return self.get_asset_connectivity(self.asset)