From c991a73632de6c8ebea44977072b0e2f2fe51d15 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sun, 23 Apr 2023 16:15:27 +0800 Subject: [PATCH 01/17] v1 --- apps/common/db/fields.py | 104 ++++++++++++++++++++++++++++++++- apps/perms/models/perm_node.py | 9 ++- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index a4a128671..97461774f 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # -import json from django.core.validators import MinValueValidator, MaxValueValidator from django.db import models @@ -32,6 +31,7 @@ __all__ = [ "PortRangeField", "BitChoices", "TreeChoices", + "JSONManyToManyField", ] @@ -274,3 +274,105 @@ class PortRangeField(models.CharField): kwargs['max_length'] = 16 super().__init__(**kwargs) self.validators.append(PortRangeValidator()) + + +from django.db.models import Q +from django.apps import apps + +from django.db import models +from django.core.exceptions import ValidationError +import json + + +class JSONManyToManyDescriptor: + def __init__(self, field): + self.field = field + self._is_setting = False + + def __get__(self, instance, owner=None): + if instance is None: + return self + + if not hasattr(instance, "_related_manager_cache"): + instance._related_manager_cache = {} + + current_value = getattr(instance, self.field.attname, {}) + + if self.field.name not in instance._related_manager_cache or instance._related_manager_cache[ + self.field.name]._is_value_stale(current_value): + manager = RelatedManager(instance, self.field) + instance._related_manager_cache[self.field.name] = manager + + return instance._related_manager_cache[self.field.name] + + def __set__(self, instance, value): + if instance is None: + return + + if not hasattr(instance, "_is_setting"): + instance._is_setting = {} + + if self.field.name not in instance._is_setting or not instance._is_setting[self.field.name]: + instance._is_setting[self.field.name] = True + manager = self.__get__(instance, instance.__class__) + manager.set(value) + serialized_value = manager.serialize() + instance.__dict__[self.field.attname] = serialized_value + instance._is_setting[self.field.name] = False + + +class JSONManyToManyField(models.JSONField): + def __init__(self, related_model, *args, **kwargs): + self.related_model = related_model + super().__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name, **kwargs): + super().contribute_to_class(cls, name, **kwargs) + setattr(cls, self.name, JSONManyToManyDescriptor(self)) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + kwargs['related_model'] = self.related_model + return name, path, args, kwargs + + def validate(self, value, model_instance): + super().validate(value, model_instance) + if not isinstance(value, list) or not all(isinstance(item, int) for item in value): + raise ValidationError("Invalid JSON data for JSONManyToManyField.") + + +class RelatedManager: + def __init__(self, instance, field): + self.instance = instance + self.field = field + + def _is_value_stale(self, current_value): + return self.serialize() != current_value + + def set(self, value): + self.field.value = value + + def serialize(self): + return self.field.value + + def _get_queryset(self): + model = apps.get_model(self.field.to) + value = self.field.value + + if value["type"] == "all": + return model.objects.all() + elif value["type"] == "ids": + return model.objects.filter(id__in=value["ids"]) + elif value["type"] == "attrs": + filters = Q() + for attr in value["attrs"]: + if attr["match"] == "exact": + filters &= Q(**{attr["attr"]: attr["value"]}) + return model.objects.filter(filters) + + def all(self): + return self._get_queryset() + + def filter(self, *args, **kwargs): + queryset = self._get_queryset() + return queryset.filter(*args, **kwargs) diff --git a/apps/perms/models/perm_node.py b/apps/perms/models/perm_node.py index e51851db9..ccd1898aa 100644 --- a/apps/perms/models/perm_node.py +++ b/apps/perms/models/perm_node.py @@ -2,12 +2,19 @@ from django.db import models from django.db.models import F, TextChoices from django.utils.translation import ugettext_lazy as _ -from assets.models import Asset, Node, FamilyMixin from accounts.models import Account +from assets.models import Asset, Node, FamilyMixin +from common.db.fields import JSONManyToManyField from common.utils import lazyproperty from orgs.mixins.models import JMSOrgBaseModel +class TestPermission2(models.Model): + name = models.CharField(max_length=128, verbose_name=_('Name')) + users = JSONManyToManyField("users.User") + assets = JSONManyToManyField("assets.Asset") + + class NodeFrom(TextChoices): granted = 'granted', 'Direct node granted' child = 'child', 'Have children node' From 378eee040202daf4a126d4955bef3ed56a85b369 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 24 Apr 2023 16:27:13 +0800 Subject: [PATCH 02/17] pref: stash 2 --- apps/common/db/fields.py | 137 ++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 59 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 97461774f..210e61719 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -278,79 +278,27 @@ class PortRangeField(models.CharField): from django.db.models import Q from django.apps import apps - from django.db import models from django.core.exceptions import ValidationError import json -class JSONManyToManyDescriptor: - def __init__(self, field): - self.field = field - self._is_setting = False - - def __get__(self, instance, owner=None): - if instance is None: - return self - - if not hasattr(instance, "_related_manager_cache"): - instance._related_manager_cache = {} - - current_value = getattr(instance, self.field.attname, {}) - - if self.field.name not in instance._related_manager_cache or instance._related_manager_cache[ - self.field.name]._is_value_stale(current_value): - manager = RelatedManager(instance, self.field) - instance._related_manager_cache[self.field.name] = manager - - return instance._related_manager_cache[self.field.name] - - def __set__(self, instance, value): - if instance is None: - return - - if not hasattr(instance, "_is_setting"): - instance._is_setting = {} - - if self.field.name not in instance._is_setting or not instance._is_setting[self.field.name]: - instance._is_setting[self.field.name] = True - manager = self.__get__(instance, instance.__class__) - manager.set(value) - serialized_value = manager.serialize() - instance.__dict__[self.field.attname] = serialized_value - instance._is_setting[self.field.name] = False - - -class JSONManyToManyField(models.JSONField): - def __init__(self, related_model, *args, **kwargs): - self.related_model = related_model - super().__init__(*args, **kwargs) - - def contribute_to_class(self, cls, name, **kwargs): - super().contribute_to_class(cls, name, **kwargs) - setattr(cls, self.name, JSONManyToManyDescriptor(self)) - - def deconstruct(self): - name, path, args, kwargs = super().deconstruct() - kwargs['related_model'] = self.related_model - return name, path, args, kwargs - - def validate(self, value, model_instance): - super().validate(value, model_instance) - if not isinstance(value, list) or not all(isinstance(item, int) for item in value): - raise ValidationError("Invalid JSON data for JSONManyToManyField.") - - class RelatedManager: def __init__(self, instance, field): self.instance = instance self.field = field def _is_value_stale(self, current_value): - return self.serialize() != current_value + return self.field.value != current_value def set(self, value): + print("set value: {} [{}] ({})".format(self, self.field, value)) + self._set_value(value) + + def _set_value(self, value): self.field.value = value + if self.instance: + self.instance.__dict__[self.field.name] = value def serialize(self): return self.field.value @@ -376,3 +324,74 @@ class RelatedManager: def filter(self, *args, **kwargs): queryset = self._get_queryset() return queryset.filter(*args, **kwargs) + + +class JSONManyToManyDescriptor: + def __init__(self, field): + print("DES Call __init__: ", field) + self.field = field + self._is_setting = False + + def __get__(self, instance, owner=None): + print("Call __get__: ", instance, id(instance)) + if instance is None: + return self + + if not hasattr(instance, "_related_manager_cache"): + instance._related_manager_cache = {} + if self.field.name not in instance._related_manager_cache: + manager = RelatedManager(instance, self.field) + instance._related_manager_cache[self.field.name] = manager + return instance._related_manager_cache[self.field.name] + + def __set__(self, instance, value): + if instance is None: + return + + if not hasattr(instance, "_related_manager_cache"): + instance._related_manager_cache = {} + + if self.field.name not in instance._related_manager_cache: + manager = self.__get__(instance, instance.__class__) + else: + manager = instance._related_manager_cache[self.field.name] + + print("manager: ", manager) + print("Call __set__: ", id(instance), value) + if isinstance(value, RelatedManager): + value = value.field.value + manager.set(value) + + +class JSONManyToManyField(models.JSONField): + def __init__(self, to, *args, **kwargs): + self.to = to + super().__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name, **kwargs): + super().contribute_to_class(cls, name, **kwargs) + setattr(cls, self.name, JSONManyToManyDescriptor(self)) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + kwargs['to'] = self.to + return name, path, args, kwargs + + def get_db_prep_value(self, value, connection, prepared=False): + if value is None: + return None + v = value.field.value + print("get_db_prep_value: ", value, v) + return json.dumps(v) + + def get_prep_value(self, value): + if value is None: + return value + v = value.field.value + print("get_prep_value: ", value, v) + return json.dumps(v) + + def validate(self, value, model_instance): + super().validate(value, model_instance) + if not isinstance(value, dict): + raise ValidationError("Invalid JSON data for JSONManyToManyField.") From 3cdb81cf4aab41aef5cc4a7e9616d7cbb015918b Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 24 Apr 2023 19:00:31 +0800 Subject: [PATCH 03/17] =?UTF-8?q?perf:=20=E6=90=9E=E5=AE=9A=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=20orm=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 210e61719..31818a438 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -287,25 +287,21 @@ class RelatedManager: def __init__(self, instance, field): self.instance = instance self.field = field + self.value = None def _is_value_stale(self, current_value): - return self.field.value != current_value + return self.value != current_value def set(self, value): - print("set value: {} [{}] ({})".format(self, self.field, value)) - self._set_value(value) - - def _set_value(self, value): - self.field.value = value - if self.instance: - self.instance.__dict__[self.field.name] = value + self.value = value + self.instance.__dict__[self.field.name] = value def serialize(self): - return self.field.value + return self.value def _get_queryset(self): model = apps.get_model(self.field.to) - value = self.field.value + value = self.value if value["type"] == "all": return model.objects.all() @@ -328,12 +324,10 @@ class RelatedManager: class JSONManyToManyDescriptor: def __init__(self, field): - print("DES Call __init__: ", field) self.field = field self._is_setting = False def __get__(self, instance, owner=None): - print("Call __get__: ", instance, id(instance)) if instance is None: return self @@ -342,7 +336,13 @@ class JSONManyToManyDescriptor: if self.field.name not in instance._related_manager_cache: manager = RelatedManager(instance, self.field) instance._related_manager_cache[self.field.name] = manager - return instance._related_manager_cache[self.field.name] + manager = instance._related_manager_cache[self.field.name] + # if self.field.name == 'users': + # print(">>> Call __get__: ", manager) + # print("Field: ", self.field) + # print("Instance: ", instance.__dict__) + # print("Current value: ", manager.value) + return manager def __set__(self, instance, value): if instance is None: @@ -356,10 +356,12 @@ class JSONManyToManyDescriptor: else: manager = instance._related_manager_cache[self.field.name] - print("manager: ", manager) - print("Call __set__: ", id(instance), value) + # if self.field.name == 'users': + # print(">>> Call __set__: ", manager, value) + # print("Field: ", self.field.name) + # print("Instance: ", instance.__dict__) if isinstance(value, RelatedManager): - value = value.field.value + value = value.value manager.set(value) @@ -380,8 +382,9 @@ class JSONManyToManyField(models.JSONField): def get_db_prep_value(self, value, connection, prepared=False): if value is None: return None - v = value.field.value - print("get_db_prep_value: ", value, v) + v = value.value + print("$$$ Get_db_prep_value: ", self.to, value, v) + print("Value field: ", value.__dict__) return json.dumps(v) def get_prep_value(self, value): From c824ae4478ea9e45459396938231b47f775e5ecf Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 24 Apr 2023 19:03:44 +0800 Subject: [PATCH 04/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 31818a438..4de393932 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -356,10 +356,6 @@ class JSONManyToManyDescriptor: else: manager = instance._related_manager_cache[self.field.name] - # if self.field.name == 'users': - # print(">>> Call __set__: ", manager, value) - # print("Field: ", self.field.name) - # print("Instance: ", instance.__dict__) if isinstance(value, RelatedManager): value = value.value manager.set(value) @@ -379,19 +375,16 @@ class JSONManyToManyField(models.JSONField): kwargs['to'] = self.to return name, path, args, kwargs - def get_db_prep_value(self, value, connection, prepared=False): - if value is None: + def get_db_prep_value(self, manager, connection, prepared=False): + if manager is None: return None - v = value.value - print("$$$ Get_db_prep_value: ", self.to, value, v) - print("Value field: ", value.__dict__) + v = manager.value return json.dumps(v) - def get_prep_value(self, value): - if value is None: - return value - v = value.field.value - print("get_prep_value: ", value, v) + def get_prep_value(self, manager): + if manager is None: + return manager + v = manager.value return json.dumps(v) def validate(self, value, model_instance): From 19d29d663798d82330574f08d707d94299ed6396 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 24 Apr 2023 19:04:47 +0800 Subject: [PATCH 05/17] perf: remove debug msg --- apps/common/db/fields.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 4de393932..978fcd865 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -337,11 +337,6 @@ class JSONManyToManyDescriptor: manager = RelatedManager(instance, self.field) instance._related_manager_cache[self.field.name] = manager manager = instance._related_manager_cache[self.field.name] - # if self.field.name == 'users': - # print(">>> Call __get__: ", manager) - # print("Field: ", self.field) - # print("Instance: ", instance.__dict__) - # print("Current value: ", manager.value) return manager def __set__(self, instance, value): From 20b7b794d84a3c1e49ced6ffb5e697a07900b280 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 25 Apr 2023 14:00:19 +0800 Subject: [PATCH 06/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20m2m=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 978fcd865..9a43061e5 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -289,16 +289,10 @@ class RelatedManager: self.field = field self.value = None - def _is_value_stale(self, current_value): - return self.value != current_value - def set(self, value): self.value = value self.instance.__dict__[self.field.name] = value - def serialize(self): - return self.value - def _get_queryset(self): model = apps.get_model(self.field.to) value = self.value From 632627db11bfbbc1c7fff56c3549af6786e4dabd Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 25 Apr 2023 16:25:00 +0800 Subject: [PATCH 07/17] =?UTF-8?q?perf:=20=E5=8E=BB=E6=8E=89=20debug=20mode?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 61 ++++++++++++++++++++++++++++------ apps/perms/models/perm_node.py | 7 ---- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 9a43061e5..7ff001e92 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -1,8 +1,13 @@ # -*- coding: utf-8 -*- # +import json + +from django.apps import apps +from django.core.exceptions import ValidationError from django.core.validators import MinValueValidator, MaxValueValidator from django.db import models +from django.db.models import Q from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ from rest_framework.utils.encoders import JSONEncoder @@ -276,13 +281,6 @@ class PortRangeField(models.CharField): self.validators.append(PortRangeValidator()) -from django.db.models import Q -from django.apps import apps -from django.db import models -from django.core.exceptions import ValidationError -import json - - class RelatedManager: def __init__(self, instance, field): self.instance = instance @@ -296,17 +294,31 @@ class RelatedManager: def _get_queryset(self): model = apps.get_model(self.field.to) value = self.value + if not value or not isinstance(value, dict): + return model.objects.none() if value["type"] == "all": return model.objects.all() - elif value["type"] == "ids": + elif value["type"] == "ids" and isinstance(value.get("ids"), list): return model.objects.filter(id__in=value["ids"]) - elif value["type"] == "attrs": + elif value["type"] == "attrs" and isinstance(value.get("attrs"), list): filters = Q() for attr in value["attrs"]: - if attr["match"] == "exact": - filters &= Q(**{attr["attr"]: attr["value"]}) + if not isinstance(attr, dict): + continue + name = attr.get('name') + val = attr.get('value') + match = attr.get('match', 'exact') + if name is None or val is None: + continue + + lookup = name + if match in ("exact", "contains", "startswith", "endswith", "regex"): + lookup = "{}__{}".format(name, match) + filters &= Q(**{lookup: val}) return model.objects.filter(filters) + else: + return model.objects.none() def all(self): return self._get_queryset() @@ -364,16 +376,43 @@ class JSONManyToManyField(models.JSONField): kwargs['to'] = self.to return name, path, args, kwargs + @staticmethod + def _check_value(val): + if not val: + return val + e = ValueError( + 'Invalid JSON data for JSONManyToManyField, should be like ' + '{"type": "all"} or {"type": "ids", "ids": []} ' + 'or {"type": "attrs", "attrs": [{"name": "ip", "match": "exact", "value": "value"}' + ) + if not isinstance(val, dict): + raise e + if val["type"] not in ["all", "ids", "attrs"]: + raise e + if val["type"] == "ids": + if not isinstance(val["ids"], list): + raise e + elif val["type"] == "attrs": + if not isinstance(val["attrs"], list): + raise e + for attr in val["attrs"]: + if not isinstance(attr, dict): + raise e + if 'name' not in attr or 'value' not in attr: + raise e + def get_db_prep_value(self, manager, connection, prepared=False): if manager is None: return None v = manager.value + self._check_value(v) return json.dumps(v) def get_prep_value(self, manager): if manager is None: return manager v = manager.value + self._check_value(v) return json.dumps(v) def validate(self, value, model_instance): diff --git a/apps/perms/models/perm_node.py b/apps/perms/models/perm_node.py index ccd1898aa..f5dc52be3 100644 --- a/apps/perms/models/perm_node.py +++ b/apps/perms/models/perm_node.py @@ -4,17 +4,10 @@ from django.utils.translation import ugettext_lazy as _ from accounts.models import Account from assets.models import Asset, Node, FamilyMixin -from common.db.fields import JSONManyToManyField from common.utils import lazyproperty from orgs.mixins.models import JMSOrgBaseModel -class TestPermission2(models.Model): - name = models.CharField(max_length=128, verbose_name=_('Name')) - users = JSONManyToManyField("users.User") - assets = JSONManyToManyField("assets.Asset") - - class NodeFrom(TextChoices): granted = 'granted', 'Direct node granted' child = 'child', 'Have children node' From 338ab5c6340947a11b460c366923037d12aca7ca Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 26 Apr 2023 19:11:53 +0800 Subject: [PATCH 08/17] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20acl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0011_auto_20230425_1704.py | 44 ++++++++++ .../migrations/0012_auto_20230426_1111.py | 42 ++++++++++ .../migrations/0013_auto_20230426_1759.py | 66 +++++++++++++++ apps/acls/models/base.py | 7 +- apps/common/db/fields.py | 81 +++++++++++++++---- 5 files changed, 221 insertions(+), 19 deletions(-) create mode 100644 apps/acls/migrations/0011_auto_20230425_1704.py create mode 100644 apps/acls/migrations/0012_auto_20230426_1111.py create mode 100644 apps/acls/migrations/0013_auto_20230426_1759.py diff --git a/apps/acls/migrations/0011_auto_20230425_1704.py b/apps/acls/migrations/0011_auto_20230425_1704.py new file mode 100644 index 000000000..2a8682129 --- /dev/null +++ b/apps/acls/migrations/0011_auto_20230425_1704.py @@ -0,0 +1,44 @@ +# Generated by Django 3.2.17 on 2023-04-25 09:04 + +from django.db import migrations + +import common.db.fields + + +class Migration(migrations.Migration): + dependencies = [ + ('acls', '0010_alter_commandfilteracl_command_groups'), + ] + + operations = [ + migrations.AddField( + model_name='commandfilteracl', + name='new_accounts', + field=common.db.fields.JSONManyToManyField(default=dict, to='assets.Account', verbose_name='Accounts'), + ), + migrations.AddField( + model_name='commandfilteracl', + name='new_assets', + field=common.db.fields.JSONManyToManyField(default=dict, to='assets.Asset', verbose_name='Assets'), + ), + migrations.AddField( + model_name='commandfilteracl', + name='new_users', + field=common.db.fields.JSONManyToManyField(default=dict, to='users.User', verbose_name='Users'), + ), + migrations.AddField( + model_name='loginassetacl', + name='new_accounts', + field=common.db.fields.JSONManyToManyField(default=dict, to='assets.Account', verbose_name='Accounts'), + ), + migrations.AddField( + model_name='loginassetacl', + name='new_assets', + field=common.db.fields.JSONManyToManyField(default=dict, to='assets.Asset', verbose_name='Assets'), + ), + migrations.AddField( + model_name='loginassetacl', + name='new_users', + field=common.db.fields.JSONManyToManyField(default=dict, to='users.User', verbose_name='Users'), + ), + ] diff --git a/apps/acls/migrations/0012_auto_20230426_1111.py b/apps/acls/migrations/0012_auto_20230426_1111.py new file mode 100644 index 000000000..23984ac30 --- /dev/null +++ b/apps/acls/migrations/0012_auto_20230426_1111.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.17 on 2023-04-26 03:11 + +from django.db import migrations + + +def migrate_base_acl_users_assets_accounts(apps, *args): + cmd_acl_model = apps.get_model('acls', 'CommandFilterACL') + login_asset_acl_model = apps.get_model('acls', 'LoginAssetACL') + + for model in [cmd_acl_model, login_asset_acl_model]: + for obj in model.objects.all(): + user_names = (obj.users or {}).get('username_group', []) + obj.new_users = { + "type": "attrs", + "attrs": [{"name": "username", "value": user_names, "match": "in"}] + } + + asset_names = (obj.assets or {}).get('name_group', []) + asset_attrs = [] + if asset_names: + asset_attrs.append({"name": "name", "value": asset_names, "match": "in"}) + asset_address = (obj.assets or {}).get('address_group', []) + if asset_address: + asset_attrs.append({"name": "address", "value": asset_address, "match": "ip_in", "rel": "or"}) + obj.new_assets = {"type": "attrs", "attrs": asset_attrs} + + account_usernames = (obj.accounts or {}).get('username_group', []) + obj.new_accounts = { + "type": "attrs", + "attrs": [{"name": "username", "value": account_usernames, "match": "in"}] + } + obj.save() + + +class Migration(migrations.Migration): + dependencies = [ + ('acls', '0011_auto_20230425_1704'), + ] + + operations = [ + migrations.RunPython(migrate_base_acl_users_assets_accounts) + ] diff --git a/apps/acls/migrations/0013_auto_20230426_1759.py b/apps/acls/migrations/0013_auto_20230426_1759.py new file mode 100644 index 000000000..56dd58446 --- /dev/null +++ b/apps/acls/migrations/0013_auto_20230426_1759.py @@ -0,0 +1,66 @@ +# Generated by Django 3.2.17 on 2023-04-26 09:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ('acls', '0012_auto_20230426_1111'), + ] + + operations = [ + migrations.RemoveField( + model_name='commandfilteracl', + name='accounts', + ), + migrations.RemoveField( + model_name='commandfilteracl', + name='assets', + ), + migrations.RemoveField( + model_name='commandfilteracl', + name='users', + ), + migrations.RemoveField( + model_name='loginassetacl', + name='accounts', + ), + migrations.RemoveField( + model_name='loginassetacl', + name='assets', + ), + migrations.RemoveField( + model_name='loginassetacl', + name='users', + ), + migrations.RenameField( + model_name='commandfilteracl', + old_name='new_accounts', + new_name='accounts', + ), + migrations.RenameField( + model_name='commandfilteracl', + old_name='new_assets', + new_name='assets', + ), + migrations.RenameField( + model_name='commandfilteracl', + old_name='new_users', + new_name='users', + ), + migrations.RenameField( + model_name='loginassetacl', + old_name='new_accounts', + new_name='accounts', + ), + migrations.RenameField( + model_name='loginassetacl', + old_name='new_assets', + new_name='assets', + ), + migrations.RenameField( + model_name='loginassetacl', + old_name='new_users', + new_name='users', + ), + ] diff --git a/apps/acls/models/base.py b/apps/acls/models/base.py index 2624e34cd..def536724 100644 --- a/apps/acls/models/base.py +++ b/apps/acls/models/base.py @@ -3,6 +3,7 @@ from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ +from common.db.fields import JSONManyToManyField from common.db.models import JMSBaseModel from common.utils import contains_ip from orgs.mixins.models import OrgModelMixin, OrgManager @@ -95,11 +96,11 @@ class BaseACL(JMSBaseModel): class UserAssetAccountBaseACL(BaseACL, OrgModelMixin): # username_group - users = models.JSONField(verbose_name=_('User')) + users = JSONManyToManyField('users.User', default=dict, verbose_name=_('Users')) # name_group, address_group - assets = models.JSONField(verbose_name=_('Asset')) + assets = JSONManyToManyField('assets.Asset', default=dict, verbose_name=_('Assets')) # username_group - accounts = models.JSONField(verbose_name=_('Account')) + accounts = JSONManyToManyField('assets.Account', default=dict, verbose_name=_('Accounts')) objects = OrgACLManager.from_queryset(UserAssetAccountACLQuerySet)() diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 7ff001e92..c80c2b84d 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # +import ipaddress import json from django.apps import apps @@ -291,6 +292,34 @@ class RelatedManager: self.value = value self.instance.__dict__[self.field.name] = value + @staticmethod + def get_ip_in_q(name, val): + q = Q() + if isinstance(val, str): + val = [val] + for ip in val: + if not ip: + continue + try: + if ip == '*': + return Q() + elif '/' in ip: + network = ipaddress.ip_network(ip) + ips = network.hosts() + q |= Q(**{"{}__in".format(name): ips}) + elif '-' in ip: + start_ip, end_ip = ip.split('-') + start_ip = ipaddress.ip_address(start_ip) + end_ip = ipaddress.ip_address(end_ip) + q |= Q(**{"{}__range".format(name): (start_ip, end_ip)}) + elif len(ip.split('.')) == 4: + q |= Q(**{"{}__exact".format(name): ip}) + else: + q |= Q(**{"{}__startswith".format(name): ip}) + except ValueError: + continue + return q + def _get_queryset(self): model = apps.get_model(self.field.to) value = self.value @@ -303,20 +332,43 @@ class RelatedManager: return model.objects.filter(id__in=value["ids"]) elif value["type"] == "attrs" and isinstance(value.get("attrs"), list): filters = Q() + excludes = Q() for attr in value["attrs"]: if not isinstance(attr, dict): continue + name = attr.get('name') val = attr.get('value') match = attr.get('match', 'exact') + rel = attr.get('rel', 'and') if name is None or val is None: continue - lookup = name - if match in ("exact", "contains", "startswith", "endswith", "regex"): + if val == '*': + filters = Q() + break + + if match == 'ip_in': + q = self.get_ip_in_q(name, val) + elif match in ("exact", "contains", "startswith", "endswith", "regex"): lookup = "{}__{}".format(name, match) - filters &= Q(**{lookup: val}) - return model.objects.filter(filters) + q = Q(**{lookup: val}) + elif match == "in" and isinstance(val, list): + if '*' not in val: + lookup = "{}__in".format(name) + q = Q(**{lookup: val}) + else: + q = Q() + else: + q = Q(**{name: val}) + + if rel == 'or': + filters |= q + elif rel == 'not': + excludes |= q + else: + filters &= q + return model.objects.filter(filters).exclude(excludes) else: return model.objects.none() @@ -401,19 +453,16 @@ class JSONManyToManyField(models.JSONField): if 'name' not in attr or 'value' not in attr: raise e - def get_db_prep_value(self, manager, connection, prepared=False): - if manager is None: - return None - v = manager.value - self._check_value(v) - return json.dumps(v) + def get_db_prep_value(self, value, connection, prepared=False): + return self.get_prep_value(value) - def get_prep_value(self, manager): - if manager is None: - return manager - v = manager.value - self._check_value(v) - return json.dumps(v) + def get_prep_value(self, value): + if value is None: + return None + if isinstance(value, RelatedManager): + value = value.value + self._check_value(value) + return json.dumps(value) def validate(self, value, model_instance): super().validate(value, model_instance) From 90090a7fc77f8243689d23ed40b59b6a748827e9 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 27 Apr 2023 14:13:40 +0800 Subject: [PATCH 09/17] =?UTF-8?q?perf:=20=E6=B7=BB=E5=8A=A0=20JSONManyToMa?= =?UTF-8?q?nyFieldSerializer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/serializers/base.py | 26 ++++---------------------- apps/common/db/fields.py | 24 ++++++++++++------------ apps/common/serializers/fields.py | 17 ++++++++++++++++- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/apps/acls/serializers/base.py b/apps/acls/serializers/base.py index dbdac67e7..ca319284f 100644 --- a/apps/acls/serializers/base.py +++ b/apps/acls/serializers/base.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from acls.models.base import ActionChoices -from common.serializers.fields import LabeledChoiceField, ObjectRelatedField +from common.serializers.fields import JSONManyToManyField, ObjectRelatedField, LabeledChoiceField from orgs.models import Organization from users.models import User @@ -52,25 +52,9 @@ class ACLAccountsSerializer(serializers.Serializer): class BaseUserAssetAccountACLSerializerMixin(serializers.Serializer): - users = ACLUsersSerializer(label=_('User')) - assets = ACLAssestsSerializer(label=_('Asset')) - accounts = ACLAccountsSerializer(label=_('Account')) - users_username_group = serializers.ListField( - source='users.username_group', read_only=True, child=serializers.CharField(), - label=_('User (username)') - ) - assets_name_group = serializers.ListField( - source='assets.name_group', read_only=True, child=serializers.CharField(), - label=_('Asset (name)') - ) - assets_address_group = serializers.ListField( - source='assets.address_group', read_only=True, child=serializers.CharField(), - label=_('Asset (address)') - ) - accounts_username_group = serializers.ListField( - source='accounts.username_group', read_only=True, child=serializers.CharField(), - label=_('Account (username)') - ) + users = JSONManyToManyField(label=_('User')) + assets = JSONManyToManyField(label=_('Asset')) + accounts = JSONManyToManyField(label=_('Account')) reviewers = ObjectRelatedField( queryset=User.objects, many=True, required=False, label=_('Reviewers') ) @@ -84,8 +68,6 @@ class BaseUserAssetAccountACLSerializerMixin(serializers.Serializer): class Meta: fields_mini = ["id", "name"] fields_small = fields_mini + [ - 'users_username_group', 'assets_address_group', 'assets_name_group', - 'accounts_username_group', "users", "accounts", "assets", "is_active", "date_created", "date_updated", "priority", "action", "comment", "created_by", "org_id", diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index c80c2b84d..50f7bb60a 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -429,29 +429,29 @@ class JSONManyToManyField(models.JSONField): return name, path, args, kwargs @staticmethod - def _check_value(val): + def check_value(val): if not val: return val - e = ValueError( - 'Invalid JSON data for JSONManyToManyField, should be like ' - '{"type": "all"} or {"type": "ids", "ids": []} ' - 'or {"type": "attrs", "attrs": [{"name": "ip", "match": "exact", "value": "value"}' - ) + e = ValueError(_( + "Invalid JSON data for JSONManyToManyField, should be like " + "{'type': 'all'} or {'type': 'ids', 'ids': []} " + "or {'type': 'attrs', 'attrs': [{'name': 'ip', 'match': 'exact', 'value': 'value'}" + )) if not isinstance(val, dict): raise e if val["type"] not in ["all", "ids", "attrs"]: - raise e + raise ValueError(_('Invalid type, should be "all", "ids" or "attrs"')) if val["type"] == "ids": if not isinstance(val["ids"], list): - raise e + raise ValueError(_("Invalid ids for ids, should be a list")) elif val["type"] == "attrs": if not isinstance(val["attrs"], list): - raise e + raise ValueError(_("Invalid attrs, should be a list of dict")) for attr in val["attrs"]: if not isinstance(attr, dict): - raise e + raise ValueError(_("Invalid attrs, should be a list of dict")) if 'name' not in attr or 'value' not in attr: - raise e + raise ValueError(_("Invalid attrs, should be has name and value")) def get_db_prep_value(self, value, connection, prepared=False): return self.get_prep_value(value) @@ -461,7 +461,7 @@ class JSONManyToManyField(models.JSONField): return None if isinstance(value, RelatedManager): value = value.value - self._check_value(value) + self.check_value(value) return json.dumps(value) def validate(self, value, model_instance): diff --git a/apps/common/serializers/fields.py b/apps/common/serializers/fields.py index 0b1e7040c..9595e09e1 100644 --- a/apps/common/serializers/fields.py +++ b/apps/common/serializers/fields.py @@ -7,7 +7,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.fields import ChoiceField, empty -from common.db.fields import TreeChoices +from common.db.fields import TreeChoices, JSONManyToManyField as ModelJSONManyToManyField from common.local import add_encrypted_field_set from common.utils import decrypt_password @@ -20,6 +20,7 @@ __all__ = [ "TreeChoicesField", "LabeledMultipleChoiceField", "PhoneField", + "JSONManyToManyField" ] @@ -216,3 +217,17 @@ class PhoneField(serializers.CharField): phone = phonenumbers.parse(value, 'CN') value = {'code': '+%s' % phone.country_code, 'phone': phone.national_number} return value + + +class JSONManyToManyField(serializers.JSONField): + def to_representation(self, value): + return value.value + + def to_internal_value(self, data): + if not data: + data = {} + try: + ModelJSONManyToManyField.check_value(data) + except ValueError as e: + raise serializers.ValidationError(e) + return super().to_internal_value(data) From 5a6e13721d43259021286f0165c8ff38ceefb4a8 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 27 Apr 2023 18:05:16 +0800 Subject: [PATCH 10/17] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20json=20m2m?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/serializers/fields.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/common/serializers/fields.py b/apps/common/serializers/fields.py index 9595e09e1..e41a90944 100644 --- a/apps/common/serializers/fields.py +++ b/apps/common/serializers/fields.py @@ -220,8 +220,17 @@ class PhoneField(serializers.CharField): class JSONManyToManyField(serializers.JSONField): - def to_representation(self, value): - return value.value + def to_representation(self, manager): + if manager is None: + return manager + value = manager.value + if not isinstance(value, dict): + return {"type": "ids", "ids": []} + if value.get("type") == "ids": + valid_ids = manager.all().values_list("id", flat=True) + valid_ids = [str(i) for i in valid_ids] + return {"type": "ids", "ids": valid_ids} + return value def to_internal_value(self, data): if not data: From a112d3c99d0cb313b0972e05f3b4af2f2b312017 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 6 May 2023 19:52:03 +0800 Subject: [PATCH 11/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20accounts=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/models/base.py | 25 +----------------------- apps/acls/serializers/base.py | 4 ++-- apps/audits/handler.py | 19 +++++++++--------- apps/common/db/fields.py | 4 +++- apps/perms/serializers/permission.py | 29 +++++++--------------------- 5 files changed, 23 insertions(+), 58 deletions(-) diff --git a/apps/acls/models/base.py b/apps/acls/models/base.py index def536724..c85af7c0b 100644 --- a/apps/acls/models/base.py +++ b/apps/acls/models/base.py @@ -95,35 +95,12 @@ class BaseACL(JMSBaseModel): class UserAssetAccountBaseACL(BaseACL, OrgModelMixin): - # username_group users = JSONManyToManyField('users.User', default=dict, verbose_name=_('Users')) - # name_group, address_group assets = JSONManyToManyField('assets.Asset', default=dict, verbose_name=_('Assets')) - # username_group - accounts = JSONManyToManyField('assets.Account', default=dict, verbose_name=_('Accounts')) + accounts = models.JSONField(default=list, verbose_name=_("Account")) objects = OrgACLManager.from_queryset(UserAssetAccountACLQuerySet)() class Meta(BaseACL.Meta): unique_together = ('name', 'org_id') abstract = True - - @classmethod - def filter_queryset(cls, user=None, asset=None, account=None, account_username=None, **kwargs): - queryset = cls.objects.all() - org_id = None - if user: - queryset = queryset.filter_user(user.username) - if account: - org_id = account.org_id - queryset = queryset.filter_account(account.username) - if account_username: - queryset = queryset.filter_account(username=account_username) - if asset: - org_id = asset.org_id - queryset = queryset.filter_asset(asset.name, asset.address) - if org_id: - kwargs['org_id'] = org_id - if kwargs: - queryset = queryset.filter(**kwargs) - return queryset diff --git a/apps/acls/serializers/base.py b/apps/acls/serializers/base.py index ca319284f..069a92a90 100644 --- a/apps/acls/serializers/base.py +++ b/apps/acls/serializers/base.py @@ -20,7 +20,7 @@ class ACLUsersSerializer(serializers.Serializer): ) -class ACLAssestsSerializer(serializers.Serializer): +class ACLAssetsSerializer(serializers.Serializer): address_group_help_text = _( "With * indicating a match all. " "Such as: " @@ -54,7 +54,7 @@ class ACLAccountsSerializer(serializers.Serializer): class BaseUserAssetAccountACLSerializerMixin(serializers.Serializer): users = JSONManyToManyField(label=_('User')) assets = JSONManyToManyField(label=_('Asset')) - accounts = JSONManyToManyField(label=_('Account')) + accounts = serializers.ListField(label=_('Account')) reviewers = ObjectRelatedField( queryset=User.objects, many=True, required=False, label=_('Reviewers') ) diff --git a/apps/audits/handler.py b/apps/audits/handler.py index 27b116488..6491c69da 100644 --- a/apps/audits/handler.py +++ b/apps/audits/handler.py @@ -1,21 +1,20 @@ +import json from datetime import datetime -from django.db import transaction from django.core.cache import cache +from django.db import transaction from django.utils.translation import ugettext_lazy as _ -from common.utils import get_request_ip, get_logger -from common.utils.timezone import as_current_tz -from common.utils.encode import Singleton from common.local import encrypted_field_set -from settings.serializers import SettingsSerializer +from common.utils import get_request_ip, get_logger +from common.utils.encode import Singleton +from common.utils.timezone import as_current_tz from jumpserver.utils import current_request -from orgs.utils import get_current_org_id from orgs.models import Organization - +from orgs.utils import get_current_org_id +from settings.serializers import SettingsSerializer from .backends import get_operate_log_storage - logger = get_logger(__name__) @@ -106,7 +105,9 @@ class OperatorLogHandler(metaclass=Singleton): return '' if isinstance(value[0], str): return ','.join(value) - return ','.join([i['value'] for i in value if i.get('value')]) + if isinstance(value[0], dict) and value[0].get('value') and isinstance(value[0]['value'], str): + return ','.join([str(i['value']) for i in value]) + return json.dumps(value) def __data_processing(self, dict_item, loop=True): encrypt_value = '******' diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 50f7bb60a..027567e4e 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -353,6 +353,8 @@ class RelatedManager: elif match in ("exact", "contains", "startswith", "endswith", "regex"): lookup = "{}__{}".format(name, match) q = Q(**{lookup: val}) + elif match == "not": + q = ~Q(**{name: val}) elif match == "in" and isinstance(val, list): if '*' not in val: lookup = "{}__in".format(name) @@ -435,7 +437,7 @@ class JSONManyToManyField(models.JSONField): e = ValueError(_( "Invalid JSON data for JSONManyToManyField, should be like " "{'type': 'all'} or {'type': 'ids', 'ids': []} " - "or {'type': 'attrs', 'attrs': [{'name': 'ip', 'match': 'exact', 'value': 'value'}" + "or {'type': 'attrs', 'attrs': [{'name': 'ip', 'match': 'exact', 'value': 'value', 'rel': 'and|or|not'}}" )) if not isinstance(val, dict): raise e diff --git a/apps/perms/serializers/permission.py b/apps/perms/serializers/permission.py index 0e5f3b80a..43876d864 100644 --- a/apps/perms/serializers/permission.py +++ b/apps/perms/serializers/permission.py @@ -44,25 +44,12 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer): model = AssetPermission fields_mini = ["id", "name"] fields_generic = [ - "accounts", - "actions", - "created_by", - "date_created", - "date_start", - "date_expired", - "is_active", - "is_expired", - "is_valid", - "comment", - "from_ticket", + "accounts", "actions", "created_by", "date_created", + "date_start", "date_expired", "is_active", "is_expired", + "is_valid", "comment", "from_ticket", ] fields_small = fields_mini + fields_generic - fields_m2m = [ - "users", - "user_groups", - "assets", - "nodes", - ] + fields_m2m = ["users", "user_groups", "assets", "nodes"] fields = fields_mini + fields_m2m + fields_generic read_only_fields = ["created_by", "date_created", "from_ticket"] extra_kwargs = { @@ -91,7 +78,8 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer): def create_accounts(self, assets): need_create_accounts = [] account_attribute = [ - 'name', 'username', 'secret_type', 'secret', 'privileged', 'is_active', 'org_id' + 'name', 'username', 'secret_type', 'secret', + 'privileged', 'is_active', 'org_id' ] for asset in assets: asset_exist_accounts = Account.objects.none() @@ -139,10 +127,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer): def setup_eager_loading(cls, queryset): """Perform necessary eager loading of data.""" queryset = queryset.prefetch_related( - "users", - "user_groups", - "assets", - "nodes", + "users", "user_groups", "assets", "nodes", ) return queryset From 1ec4cbdf38c7c3283c102b8d66cef1dae1b3f6b9 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 8 May 2023 14:09:44 +0800 Subject: [PATCH 12/17] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20m2m=20json?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/api/login_asset_check.py | 3 +- apps/acls/models/base.py | 49 ++--------------------------- apps/acls/models/login_asset_acl.py | 2 -- apps/common/db/models.py | 2 -- 4 files changed, 5 insertions(+), 51 deletions(-) diff --git a/apps/acls/api/login_asset_check.py b/apps/acls/api/login_asset_check.py index a593f0c27..f272dd1d4 100644 --- a/apps/acls/api/login_asset_check.py +++ b/apps/acls/api/login_asset_check.py @@ -37,7 +37,8 @@ class LoginAssetCheckAPI(CreateAPIView): 'account_username': self.serializer.validated_data.get('account_username'), 'action': LoginAssetACL.ActionChoices.review } - acl = LoginAssetACL.filter_queryset(**kwargs).valid().first() + acl = LoginAssetACL.objects.filter(**kwargs).valid().first() + if acl: need_review = True response_data = self._get_response_data_of_need_review(acl) diff --git a/apps/acls/models/base.py b/apps/acls/models/base.py index c85af7c0b..80f0affe6 100644 --- a/apps/acls/models/base.py +++ b/apps/acls/models/base.py @@ -1,19 +1,13 @@ from django.core.validators import MinValueValidator, MaxValueValidator from django.db import models -from django.db.models import Q from django.utils.translation import gettext_lazy as _ from common.db.fields import JSONManyToManyField from common.db.models import JMSBaseModel -from common.utils import contains_ip -from orgs.mixins.models import OrgModelMixin, OrgManager +from orgs.mixins.models import OrgModelMixin __all__ = [ - 'ACLManager', - 'BaseACL', - 'BaseACLQuerySet', - 'UserAssetAccountBaseACL', - 'UserAssetAccountACLQuerySet' + 'BaseACL', 'UserAssetAccountBaseACL', ] @@ -37,41 +31,6 @@ class BaseACLQuerySet(models.QuerySet): return self.inactive() -class UserAssetAccountACLQuerySet(BaseACLQuerySet): - def filter_user(self, username): - q = Q(users__username_group__contains=username) | \ - Q(users__username_group__contains='*') - return self.filter(q) - - def filter_asset(self, name=None, address=None): - queryset = self.filter() - if name: - q = Q(assets__name_group__contains=name) | \ - Q(assets__name_group__contains='*') - queryset = queryset.filter(q) - if address: - ids = [ - q.id for q in queryset - if contains_ip(address, q.assets.get('address_group', [])) - ] - queryset = queryset.filter(id__in=ids) - return queryset - - def filter_account(self, username): - q = Q(accounts__username_group__contains=username) | \ - Q(accounts__username_group__contains='*') - return self.filter(q) - - -class ACLManager(models.Manager): - def valid(self): - return self.get_queryset().valid() - - -class OrgACLManager(OrgManager, ACLManager): - pass - - class BaseACL(JMSBaseModel): name = models.CharField(max_length=128, verbose_name=_('Name')) priority = models.IntegerField( @@ -84,7 +43,7 @@ class BaseACL(JMSBaseModel): is_active = models.BooleanField(default=True, verbose_name=_("Active")) ActionChoices = ActionChoices - objects = ACLManager.from_queryset(BaseACLQuerySet)() + objects = BaseACLQuerySet.as_manager() class Meta: ordering = ('priority', 'date_updated', 'name') @@ -99,8 +58,6 @@ class UserAssetAccountBaseACL(BaseACL, OrgModelMixin): assets = JSONManyToManyField('assets.Asset', default=dict, verbose_name=_('Assets')) accounts = models.JSONField(default=list, verbose_name=_("Account")) - objects = OrgACLManager.from_queryset(UserAssetAccountACLQuerySet)() - class Meta(BaseACL.Meta): unique_together = ('name', 'org_id') abstract = True diff --git a/apps/acls/models/login_asset_acl.py b/apps/acls/models/login_asset_acl.py index 609491e8d..fc5da088b 100644 --- a/apps/acls/models/login_asset_acl.py +++ b/apps/acls/models/login_asset_acl.py @@ -1,11 +1,9 @@ from django.utils.translation import ugettext_lazy as _ - from .base import UserAssetAccountBaseACL class LoginAssetACL(UserAssetAccountBaseACL): - class Meta(UserAssetAccountBaseACL.Meta): verbose_name = _('Login asset acl') abstract = False diff --git a/apps/common/db/models.py b/apps/common/db/models.py index 4d292827a..d01a37266 100644 --- a/apps/common/db/models.py +++ b/apps/common/db/models.py @@ -10,7 +10,6 @@ """ import uuid -from functools import reduce from django.db import models from django.db import transaction @@ -55,7 +54,6 @@ def output_as_string(field_name): class MultiTableChildQueryset(QuerySet): - def bulk_create(self, objs, batch_size=None): assert batch_size is None or batch_size > 0 if not objs: From 7c850a8a1eebd11f70e968b3678d665cd1f39e6f Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 12 May 2023 19:16:55 +0800 Subject: [PATCH 13/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20json=20field?= =?UTF-8?q?=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/api/login_asset_check.py | 21 ++++-- .../migrations/0012_auto_20230426_1111.py | 2 +- apps/common/db/fields.py | 68 ++++++++++++++++--- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/apps/acls/api/login_asset_check.py b/apps/acls/api/login_asset_check.py index f272dd1d4..3c157a1cc 100644 --- a/apps/acls/api/login_asset_check.py +++ b/apps/acls/api/login_asset_check.py @@ -1,6 +1,7 @@ from rest_framework.generics import CreateAPIView from rest_framework.response import Response +from common.db.fields import JSONManyToManyField from common.utils import reverse, lazyproperty from orgs.utils import tmp_to_org from .. import serializers @@ -30,14 +31,20 @@ class LoginAssetCheckAPI(CreateAPIView): return serializer def check_review(self): + user = self.serializer.user + asset = self.serializer.asset + + # 用户满足的 acls + queryset = LoginAssetACL.objects.all() + q = JSONManyToManyField.get_filter_q(LoginAssetACL, 'users', user) + queryset = queryset.filter(q) + q = JSONManyToManyField.get_filter_q(LoginAssetACL, 'assets', asset) + queryset = queryset.filter(q) + account_username = self.serializer.validated_data.get('account_username') + queryset = queryset.filter(accounts__contains=account_username) + with tmp_to_org(self.serializer.asset.org): - kwargs = { - 'user': self.serializer.user, - 'asset': self.serializer.asset, - 'account_username': self.serializer.validated_data.get('account_username'), - 'action': LoginAssetACL.ActionChoices.review - } - acl = LoginAssetACL.objects.filter(**kwargs).valid().first() + acl = queryset.order_by('priority').valid().first() if acl: need_review = True diff --git a/apps/acls/migrations/0012_auto_20230426_1111.py b/apps/acls/migrations/0012_auto_20230426_1111.py index 23984ac30..277905fcd 100644 --- a/apps/acls/migrations/0012_auto_20230426_1111.py +++ b/apps/acls/migrations/0012_auto_20230426_1111.py @@ -21,7 +21,7 @@ def migrate_base_acl_users_assets_accounts(apps, *args): asset_attrs.append({"name": "name", "value": asset_names, "match": "in"}) asset_address = (obj.assets or {}).get('address_group', []) if asset_address: - asset_attrs.append({"name": "address", "value": asset_address, "match": "ip_in", "rel": "or"}) + asset_attrs.append({"name": "address", "value": asset_address, "match": "ip_in"}) obj.new_assets = {"type": "attrs", "attrs": asset_attrs} account_usernames = (obj.accounts or {}).get('username_group', []) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 027567e4e..70c14c4f2 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -3,6 +3,7 @@ import ipaddress import json +import re from django.apps import apps from django.core.exceptions import ValidationError @@ -344,10 +345,6 @@ class RelatedManager: if name is None or val is None: continue - if val == '*': - filters = Q() - break - if match == 'ip_in': q = self.get_ip_in_q(name, val) elif match in ("exact", "contains", "startswith", "endswith", "regex"): @@ -362,7 +359,10 @@ class RelatedManager: else: q = Q() else: - q = Q(**{name: val}) + if val == '*': + q = Q() + else: + q = Q(**{name: val}) if rel == 'or': filters |= q @@ -415,6 +415,59 @@ class JSONManyToManyDescriptor: value = value.value manager.set(value) + def test_is(self): + print("Self.field is", self.field) + print("Self.field to", self.field.to) + print("Self.field model", self.field.model) + print("Self.field column", self.field.column) + print("Self.field to", self.field.__dict__) + + @staticmethod + def attr_to_regex(attr): + """将属性规则转换为正则表达式""" + name, value, match = attr['name'], attr['value'], attr['match'] + if match == 'contains': + return r'.*{}.*'.format(escape_regex(value)) + elif match == 'startswith': + return r'^{}.*'.format(escape_regex(value)) + elif match == 'endswith': + return r'.*{}$'.format(escape_regex(value)) + elif match == 'regex': + return value + elif match == 'not': + return r'^(?!^{}$)'.format(escape_regex(value)) + elif match == 'in': + values = '|'.join(map(escape_regex, value)) + return r'^(?:{})$'.format(values) + else: + return r'^{}$'.format(escape_regex(value)) + + def is_match(self, attr_dict, attr_rules): + for rule in attr_rules: + value = attr_dict.get(rule['name'], '') + regex = self.attr_to_regex(rule) + if not re.match(regex, value): + return False + return True + + def get_filter_q(self, instance): + model_cls = self.field.model + field_name = self.field.column + q = Q(users__type='all') | Q(users__type='ids', users__ids__contains=[str(instance.id)]) + queryset_id_attrs = model_cls.objects \ + .filter(**{'{}__type'.format(field_name): 'attrs'}) \ + .values_list('id', '{}__attrs'.format(field_name)) + instance_attr = {k: v for k, v in instance.__dict__.items() if not k.startswith('_')} + ids = [str(_id) for _id, attr_rules in queryset_id_attrs if self.is_match(instance_attr, attr_rules)] + if ids: + q |= Q(id__in=ids) + return q + + +def escape_regex(s): + """转义字符串中的正则表达式特殊字符""" + return re.sub('[.*+?^${}()|[\\]]', r'\\\g<0>', s) + class JSONManyToManyField(models.JSONField): def __init__(self, to, *args, **kwargs): @@ -455,18 +508,15 @@ class JSONManyToManyField(models.JSONField): if 'name' not in attr or 'value' not in attr: raise ValueError(_("Invalid attrs, should be has name and value")) - def get_db_prep_value(self, value, connection, prepared=False): - return self.get_prep_value(value) - def get_prep_value(self, value): if value is None: return None if isinstance(value, RelatedManager): value = value.value - self.check_value(value) return json.dumps(value) def validate(self, value, model_instance): super().validate(value, model_instance) if not isinstance(value, dict): raise ValidationError("Invalid JSON data for JSONManyToManyField.") + self.check_value(value) From 4e5ab5a605000c52346d5c987315f0fbc13083e5 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 18 May 2023 13:14:32 +0800 Subject: [PATCH 14/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E7=9A=84=20q?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/api/login_acl.py | 3 +- apps/acls/api/login_asset_check.py | 5 +- apps/common/db/fields.py | 219 ++++++++++++++++++----------- apps/common/utils/ip/utils.py | 18 +++ apps/users/models/user.py | 28 +++- 5 files changed, 181 insertions(+), 92 deletions(-) diff --git a/apps/acls/api/login_acl.py b/apps/acls/api/login_acl.py index 806ffb343..07aa7f46e 100644 --- a/apps/acls/api/login_acl.py +++ b/apps/acls/api/login_acl.py @@ -1,7 +1,7 @@ from common.api import JMSBulkModelViewSet -from ..models import LoginACL from .. import serializers from ..filters import LoginAclFilter +from ..models import LoginACL __all__ = ['LoginACLViewSet'] @@ -11,4 +11,3 @@ class LoginACLViewSet(JMSBulkModelViewSet): filterset_class = LoginAclFilter search_fields = ('name',) serializer_class = serializers.LoginACLSerializer - diff --git a/apps/acls/api/login_asset_check.py b/apps/acls/api/login_asset_check.py index 3c157a1cc..bb61a9043 100644 --- a/apps/acls/api/login_asset_check.py +++ b/apps/acls/api/login_asset_check.py @@ -1,7 +1,6 @@ from rest_framework.generics import CreateAPIView from rest_framework.response import Response -from common.db.fields import JSONManyToManyField from common.utils import reverse, lazyproperty from orgs.utils import tmp_to_org from .. import serializers @@ -36,9 +35,9 @@ class LoginAssetCheckAPI(CreateAPIView): # 用户满足的 acls queryset = LoginAssetACL.objects.all() - q = JSONManyToManyField.get_filter_q(LoginAssetACL, 'users', user) + q = LoginAssetACL.users.get_filter_q(LoginAssetACL, 'users', user) queryset = queryset.filter(q) - q = JSONManyToManyField.get_filter_q(LoginAssetACL, 'assets', asset) + q = LoginAssetACL.assets.get_filter_q(LoginAssetACL, 'assets', asset) queryset = queryset.filter(q) account_username = self.serializer.validated_data.get('account_username') queryset = queryset.filter(accounts__contains=account_username) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 70c14c4f2..d2087f5d4 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -3,19 +3,20 @@ import ipaddress import json +import logging import re from django.apps import apps from django.core.exceptions import ValidationError from django.core.validators import MinValueValidator, MaxValueValidator from django.db import models -from django.db.models import Q +from django.db.models import Q, Manager from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ from rest_framework.utils.encoders import JSONEncoder from common.local import add_encrypted_field_set -from common.utils import signer, crypto +from common.utils import signer, crypto, contains_ip from .validators import PortRangeValidator __all__ = [ @@ -321,58 +322,82 @@ class RelatedManager: continue return q + def _get_filter_attrs_q(self, value, to_model): + filters = Q() + # 特殊情况有这几种， + # 1. 像 资产中的 type 和 category，集成自 Platform。所以不能直接查询 + # 2. 像 资产中的 nodes，不是简单的 m2m，是树 的关系 + # 3. 像 用户中的 orgs 也不是简单的 m2m，也是计算出来的 + # get_filter_{}_attr_q 处理复杂的 + custom_attr_filter = getattr(to_model, "get_json_filter_attr_q", None) + for attr in value["attrs"]: + if not isinstance(attr, dict): + continue + + name = attr.get('name') + val = attr.get('value') + match = attr.get('match', 'exact') + if name is None or val is None: + continue + + print("Has custom filter: {}".format(custom_attr_filter)) + if custom_attr_filter: + custom_filter_q = custom_attr_filter(name, val, match) + print("Custom filter: {}".format(custom_filter_q)) + if custom_filter_q: + filters &= custom_filter_q + continue + + if match == 'ip_in': + q = self.get_ip_in_q(name, val) + elif match in ("exact", "contains", "startswith", "endswith", "regex", "gte", "lte", "gt", "lt"): + lookup = "{}__{}".format(name, match) + q = Q(**{lookup: val}) + elif match == "not": + q = ~Q(**{name: val}) + elif match == "m2m": + if not isinstance(val, list): + val = [val] + q = Q(**{"{}__in".format(name): val}) + elif match == "in" and isinstance(val, list): + if '*' not in val: + lookup = "{}__in".format(name) + q = Q(**{lookup: val}) + else: + q = Q() + else: + if val == '*': + q = Q() + else: + q = Q(**{name: val}) + + filters &= q + return filters + def _get_queryset(self): - model = apps.get_model(self.field.to) + to_model = apps.get_model(self.field.to) value = self.value + if hasattr(to_model, "get_queryset"): + queryset = to_model.get_queryset() + else: + queryset = to_model.objects.all() + if not value or not isinstance(value, dict): - return model.objects.none() + return queryset.none() if value["type"] == "all": - return model.objects.all() + return queryset elif value["type"] == "ids" and isinstance(value.get("ids"), list): - return model.objects.filter(id__in=value["ids"]) + return queryset.filter(id__in=value["ids"]) elif value["type"] == "attrs" and isinstance(value.get("attrs"), list): - filters = Q() - excludes = Q() - for attr in value["attrs"]: - if not isinstance(attr, dict): - continue - - name = attr.get('name') - val = attr.get('value') - match = attr.get('match', 'exact') - rel = attr.get('rel', 'and') - if name is None or val is None: - continue - - if match == 'ip_in': - q = self.get_ip_in_q(name, val) - elif match in ("exact", "contains", "startswith", "endswith", "regex"): - lookup = "{}__{}".format(name, match) - q = Q(**{lookup: val}) - elif match == "not": - q = ~Q(**{name: val}) - elif match == "in" and isinstance(val, list): - if '*' not in val: - lookup = "{}__in".format(name) - q = Q(**{lookup: val}) - else: - q = Q() - else: - if val == '*': - q = Q() - else: - q = Q(**{name: val}) - - if rel == 'or': - filters |= q - elif rel == 'not': - excludes |= q - else: - filters &= q - return model.objects.filter(filters).exclude(excludes) + q = self._get_filter_attrs_q(value, to_model) + return queryset.filter(q) else: - return model.objects.none() + return queryset.none() + + def get_attr_q(self): + q = self._get_filter_attrs_q(self.value) + return q def all(self): return self._get_queryset() @@ -415,40 +440,68 @@ class JSONManyToManyDescriptor: value = value.value manager.set(value) - def test_is(self): - print("Self.field is", self.field) - print("Self.field to", self.field.to) - print("Self.field model", self.field.model) - print("Self.field column", self.field.column) - print("Self.field to", self.field.__dict__) + def is_match(self, obj, attr_rules): + # m2m 的情况 + # 自定义的情况：比如 nodes, category + res = True + to_model = apps.get_model(self.field.to) + src_model = self.field.model + field_name = self.field.name + custom_attr_filter = getattr(src_model, "get_filter_{}_attr_q".format(field_name), None) - @staticmethod - def attr_to_regex(attr): - """将属性规则转换为正则表达式""" - name, value, match = attr['name'], attr['value'], attr['match'] - if match == 'contains': - return r'.*{}.*'.format(escape_regex(value)) - elif match == 'startswith': - return r'^{}.*'.format(escape_regex(value)) - elif match == 'endswith': - return r'.*{}$'.format(escape_regex(value)) - elif match == 'regex': - return value - elif match == 'not': - return r'^(?!^{}$)'.format(escape_regex(value)) - elif match == 'in': - values = '|'.join(map(escape_regex, value)) - return r'^(?:{})$'.format(values) - else: - return r'^{}$'.format(escape_regex(value)) - - def is_match(self, attr_dict, attr_rules): + custom_q = Q() for rule in attr_rules: - value = attr_dict.get(rule['name'], '') - regex = self.attr_to_regex(rule) - if not re.match(regex, value): - return False - return True + value = getattr(obj, rule['name'], '') + rule_value = rule.get('value', '') + rule_match = rule.get('match', 'exact') + + if custom_attr_filter: + q = custom_attr_filter(rule['name'], rule_value, rule_match) + if q: + custom_q &= q + continue + + if rule_match == 'in': + res &= value in rule_value + elif rule_match == 'exact': + res &= value == rule_value + elif rule_match == 'contains': + res &= rule_value in value + elif rule_match == 'startswith': + res &= str(value).startswith(str(rule_value)) + elif rule_match == 'endswith': + res &= str(value).endswith(str(rule_value)) + elif rule_match == 'regex': + res &= re.match(rule_value, value) + elif rule_match == 'not': + res &= value != rule_value + elif rule['match'] == 'gte': + res &= value >= rule_value + elif rule['match'] == 'lte': + res &= value <= rule_value + elif rule['match'] == 'gt': + res &= value > rule_value + elif rule['match'] == 'lt': + res &= value < rule_value + elif rule['match'] == 'ip_in': + if isinstance(rule_value, str): + rule_value = [rule_value] + res &= contains_ip(value, rule_value) + elif rule['match'] == 'm2m': + if isinstance(value, Manager): + value = value.values_list('id', flat=True) + value = set(map(str, value)) + rule_value = set(map(str, rule_value)) + res &= rule_value.issubset(value) + else: + logging.error("unknown match: {}".format(rule['match'])) + res &= False + + if not res: + return res + if custom_q: + res &= to_model.objects.filter(custom_q).filter(id=obj.id).exists() + return res def get_filter_q(self, instance): model_cls = self.field.model @@ -457,18 +510,12 @@ class JSONManyToManyDescriptor: queryset_id_attrs = model_cls.objects \ .filter(**{'{}__type'.format(field_name): 'attrs'}) \ .values_list('id', '{}__attrs'.format(field_name)) - instance_attr = {k: v for k, v in instance.__dict__.items() if not k.startswith('_')} - ids = [str(_id) for _id, attr_rules in queryset_id_attrs if self.is_match(instance_attr, attr_rules)] + ids = [str(_id) for _id, attr_rules in queryset_id_attrs if self.is_match(instance, attr_rules)] if ids: q |= Q(id__in=ids) return q -def escape_regex(s): - """转义字符串中的正则表达式特殊字符""" - return re.sub('[.*+?^${}()|[\\]]', r'\\\g<0>', s) - - class JSONManyToManyField(models.JSONField): def __init__(self, to, *args, **kwargs): self.to = to @@ -490,7 +537,7 @@ class JSONManyToManyField(models.JSONField): e = ValueError(_( "Invalid JSON data for JSONManyToManyField, should be like " "{'type': 'all'} or {'type': 'ids', 'ids': []} " - "or {'type': 'attrs', 'attrs': [{'name': 'ip', 'match': 'exact', 'value': 'value', 'rel': 'and|or|not'}}" + "or {'type': 'attrs', 'attrs': [{'name': 'ip', 'match': 'exact', 'value': '1.1.1.1'}}" )) if not isinstance(val, dict): raise e diff --git a/apps/common/utils/ip/utils.py b/apps/common/utils/ip/utils.py index e5d43911e..14851ff27 100644 --- a/apps/common/utils/ip/utils.py +++ b/apps/common/utils/ip/utils.py @@ -1,3 +1,4 @@ +import ipaddress import socket from ipaddress import ip_network, ip_address @@ -75,6 +76,23 @@ def contains_ip(ip, ip_group): return False +def is_ip(self, ip, rule_value): + if rule_value == '*': + return True + elif '/' in rule_value: + network = ipaddress.ip_network(rule_value) + return ip in network.hosts() + elif '-' in rule_value: + start_ip, end_ip = rule_value.split('-') + start_ip = ipaddress.ip_address(start_ip) + end_ip = ipaddress.ip_address(end_ip) + return start_ip <= ip <= end_ip + elif len(rule_value.split('.')) == 4: + return ip == rule_value + else: + return ip.startswith(rule_value) + + def get_ip_city(ip): if not ip or not isinstance(ip, str): return _("Invalid address") diff --git a/apps/users/models/user.py b/apps/users/models/user.py index b487365e5..5d1edb036 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -668,7 +668,33 @@ class MFAMixin: return backend -class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser): +class JSONFilterMixin: + """ + users = JSONManyToManyField('users.User', blank=True, null=True) + """ + + @staticmethod + def get_json_filter_attr_q(name, value, match): + from rbac.models import RoleBinding + from orgs.utils import current_org + + if name == 'system_roles': + user_id = RoleBinding.objects \ + .filter(role__in=value, scope='system') \ + .values_list('user_id', flat=True) + return models.Q(id__in=user_id) + elif name == 'org_roles': + kwargs = dict(role__in=value, scope='org') + if not current_org.is_root(): + kwargs['org_id'] = current_org.id + + user_id = RoleBinding.objects.filter(**kwargs) \ + .values_list('user_id', flat=True) + return models.Q(id__in=user_id) + return None + + +class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, JSONFilterMixin, AbstractUser): class Source(models.TextChoices): local = 'local', _('Local') ldap = 'ldap', 'LDAP/AD' From ebaa8d26377fca87f21dc22f42fd6b479233f0d8 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 18 May 2023 17:31:40 +0800 Subject: [PATCH 15/17] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20json=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 50 +++++++++++++++++++++----------------- apps/common/drf/filters.py | 25 +++++++++++++++++++ apps/users/api/user.py | 10 +++----- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index d2087f5d4..6039e943a 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -294,6 +294,29 @@ class RelatedManager: self.value = value self.instance.__dict__[self.field.name] = value + @classmethod + def get_filter_q(cls, value, to_model): + if not value or not isinstance(value, dict): + return Q() + + if value["type"] == "all": + return Q() + elif value["type"] == "ids" and isinstance(value.get("ids"), list): + return Q(id__in=value["ids"]) + elif value["type"] == "attrs" and isinstance(value.get("attrs"), list): + return cls._get_filter_attrs_q(value, to_model) + else: + return Q() + + @classmethod + def filter_queryset_by_model(cls, value, to_model): + if hasattr(to_model, "get_queryset"): + queryset = to_model.get_queryset() + else: + queryset = to_model.objects.all() + q = cls.get_filter_q(value, to_model) + return queryset.filter(q) + @staticmethod def get_ip_in_q(name, val): q = Q() @@ -322,7 +345,8 @@ class RelatedManager: continue return q - def _get_filter_attrs_q(self, value, to_model): + @classmethod + def _get_filter_attrs_q(cls, value, to_model): filters = Q() # 特殊情况有这几种， # 1. 像 资产中的 type 和 category，集成自 Platform。所以不能直接查询 @@ -340,16 +364,14 @@ class RelatedManager: if name is None or val is None: continue - print("Has custom filter: {}".format(custom_attr_filter)) if custom_attr_filter: custom_filter_q = custom_attr_filter(name, val, match) - print("Custom filter: {}".format(custom_filter_q)) if custom_filter_q: filters &= custom_filter_q continue if match == 'ip_in': - q = self.get_ip_in_q(name, val) + q = cls.get_ip_in_q(name, val) elif match in ("exact", "contains", "startswith", "endswith", "regex", "gte", "lte", "gt", "lt"): lookup = "{}__{}".format(name, match) q = Q(**{lookup: val}) @@ -377,26 +399,10 @@ class RelatedManager: def _get_queryset(self): to_model = apps.get_model(self.field.to) value = self.value - if hasattr(to_model, "get_queryset"): - queryset = to_model.get_queryset() - else: - queryset = to_model.objects.all() - - if not value or not isinstance(value, dict): - return queryset.none() - - if value["type"] == "all": - return queryset - elif value["type"] == "ids" and isinstance(value.get("ids"), list): - return queryset.filter(id__in=value["ids"]) - elif value["type"] == "attrs" and isinstance(value.get("attrs"), list): - q = self._get_filter_attrs_q(value, to_model) - return queryset.filter(q) - else: - return queryset.none() + return self.filter_queryset_by_model(value, to_model) def get_attr_q(self): - q = self._get_filter_attrs_q(self.value) + q = self._get_filter_attrs_q(self.value, apps.get_model(self.field.to)) return q def all(self): diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index 949260a47..6278efccf 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # +import base64 +import json import logging from django.core.cache import cache @@ -18,6 +20,8 @@ __all__ = [ "BaseFilterSet" ] +from common.db.fields import RelatedManager + class BaseFilterSet(drf_filters.FilterSet): def do_nothing(self, queryset, name, value): @@ -183,3 +187,24 @@ class UUIDInFilter(drf_filters.BaseInFilter, drf_filters.UUIDFilter): class NumberInFilter(drf_filters.BaseInFilter, drf_filters.NumberFilter): pass + + +class AttrRulesFilter(filters.BaseFilterBackend): + def get_schema_fields(self, view): + return [ + coreapi.Field( + name='attr_rules', location='query', required=False, + type='string', example='/api/v1/users/users?attr_rules=jsonbase64', + description='Filter by json like {"type": "attrs", "attrs": []} to base64' + ) + ] + + def filter_queryset(self, request, queryset, view): + attr_rules = request.query_params.get('attr_rules') + if not attr_rules: + return queryset + + attr_rules = base64.b64decode(attr_rules.encode('utf-8')) + attr_rules = json.loads(attr_rules) + q = RelatedManager.get_filter_q(attr_rules, queryset.model) + return queryset.filter(q) diff --git a/apps/users/api/user.py b/apps/users/api/user.py index d1eee8083..26a8e9d05 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -7,8 +7,8 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet -from common.api import CommonApiMixin -from common.api import SuggestionMixin +from common.api import CommonApiMixin, SuggestionMixin +from common.drf.filters import AttrRulesFilter from common.utils import get_logger from orgs.utils import current_org, tmp_to_root_org from rbac.models import Role, RoleBinding @@ -18,10 +18,7 @@ from .. import serializers from ..filters import UserFilter from ..models import User from ..notifications import ResetMFAMsg -from ..serializers import ( - UserSerializer, - MiniUserSerializer, InviteSerializer -) +from ..serializers import UserSerializer, MiniUserSerializer, InviteSerializer from ..signals import post_user_create logger = get_logger(__name__) @@ -33,6 +30,7 @@ __all__ = [ class UserViewSet(CommonApiMixin, UserQuerysetMixin, SuggestionMixin, BulkModelViewSet): filterset_class = UserFilter + extra_filter_backends = [AttrRulesFilter] search_fields = ('username', 'email', 'name') serializer_classes = { 'default': UserSerializer, From a261d69cd2ce925b186300ed13d65cd10a81c51b Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 18 May 2023 21:34:19 +0800 Subject: [PATCH 16/17] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20m2m=20json?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/asset/asset.py | 7 +++++-- apps/assets/models/asset/common.py | 28 +++++++++++++++++++++++++++- apps/assets/models/node.py | 13 +++++++++++++ apps/common/db/fields.py | 1 + apps/common/drf/filters.py | 16 ++++++++++++---- apps/users/api/user.py | 4 ++-- 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index ba048e236..71d912188 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -15,7 +15,7 @@ from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBack from assets.models import Asset, Gateway, Platform from assets.tasks import test_assets_connectivity_manual, update_assets_hardware_info_manual from common.api import SuggestionMixin -from common.drf.filters import BaseFilterSet +from common.drf.filters import BaseFilterSet, AttrRulesFilterBackend from common.utils import get_logger, is_uuid from orgs.mixins import generics from orgs.mixins.api import OrgBulkModelViewSet @@ -110,7 +110,10 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): ("spec_info", "assets.view_asset"), ("gathered_info", "assets.view_asset"), ) - extra_filter_backends = [LabelFilterBackend, IpInFilterBackend, NodeFilterBackend] + extra_filter_backends = [ + LabelFilterBackend, IpInFilterBackend, + NodeFilterBackend, AttrRulesFilterBackend + ] def get_serializer_class(self): cls = super().get_serializer_class() diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index b39119b81..6e6d6b836 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -6,6 +6,7 @@ import logging from collections import defaultdict from django.db import models +from django.db.models import Q from django.forms import model_to_dict from django.utils.translation import ugettext_lazy as _ @@ -116,7 +117,32 @@ class Protocol(models.Model): return self.asset_platform_protocol.get('public', True) -class Asset(NodesRelationMixin, AbsConnectivity, JMSOrgBaseModel): +class JSONFilterMixin: + @staticmethod + def get_json_filter_attr_q(name, value, match): + """ + :param name: 属性名称 + :param value: 定义的结果 + :param match: 匹配方式 + :return: + """ + from ..node import Node + if not isinstance(value, (list, tuple)): + value = [value] + if name == 'nodes': + nodes = Node.objects.filter(id__in=value) + children = Node.get_nodes_all_children(nodes, with_self=True).values_list('id', flat=True) + return Q(nodes__in=children) + elif name == 'category': + return Q(platform__category__in=value) + elif name == 'type': + return Q(platform__type__in=value) + elif name == 'protocols': + return Q(protocols__name__in=value) + return None + + +class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel): Category = const.Category Type = const.AllTypes diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 32bfcaa09..3a729ba9f 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -63,6 +63,19 @@ class FamilyMixin: pattern += r'|^{0}$'.format(key) return pattern + @classmethod + def get_nodes_children_key_pattern(cls, nodes, with_self=True): + keys = [i.key for i in nodes] + keys = cls.clean_children_keys(keys) + patterns = [cls.get_node_all_children_key_pattern(key) for key in keys] + patterns = '|'.join(patterns) + return patterns + + @classmethod + def get_nodes_all_children(cls, nodes, with_self=True): + pattern = cls.get_nodes_children_key_pattern(nodes, with_self=with_self) + return Node.objects.filter(key__iregex=pattern) + @classmethod def get_node_children_key_pattern(cls, key, with_self=True): pattern = r'^{0}:[0-9]+$'.format(key) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 6039e943a..3897a8c5e 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -315,6 +315,7 @@ class RelatedManager: else: queryset = to_model.objects.all() q = cls.get_filter_q(value, to_model) + print("Q: ", q) return queryset.filter(q) @staticmethod diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index 6278efccf..cfdff0a6e 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -189,7 +189,7 @@ class NumberInFilter(drf_filters.BaseInFilter, drf_filters.NumberFilter): pass -class AttrRulesFilter(filters.BaseFilterBackend): +class AttrRulesFilterBackend(filters.BaseFilterBackend): def get_schema_fields(self, view): return [ coreapi.Field( @@ -204,7 +204,15 @@ class AttrRulesFilter(filters.BaseFilterBackend): if not attr_rules: return queryset - attr_rules = base64.b64decode(attr_rules.encode('utf-8')) - attr_rules = json.loads(attr_rules) + try: + attr_rules = base64.b64decode(attr_rules.encode('utf-8')) + except Exception: + raise ValidationError({'attr_rules': 'attr_rules should be base64'}) + try: + attr_rules = json.loads(attr_rules) + except Exception: + raise ValidationError({'attr_rules': 'attr_rules should be json'}) + + logging.debug('attr_rules: %s', attr_rules) q = RelatedManager.get_filter_q(attr_rules, queryset.model) - return queryset.filter(q) + return queryset.filter(q).distinct() diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 26a8e9d05..c7882f6b2 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -8,7 +8,7 @@ from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet from common.api import CommonApiMixin, SuggestionMixin -from common.drf.filters import AttrRulesFilter +from common.drf.filters import AttrRulesFilterBackend from common.utils import get_logger from orgs.utils import current_org, tmp_to_root_org from rbac.models import Role, RoleBinding @@ -30,7 +30,7 @@ __all__ = [ class UserViewSet(CommonApiMixin, UserQuerysetMixin, SuggestionMixin, BulkModelViewSet): filterset_class = UserFilter - extra_filter_backends = [AttrRulesFilter] + extra_filter_backends = [AttrRulesFilterBackend] search_fields = ('username', 'email', 'name') serializer_classes = { 'default': UserSerializer, From 197364d42d02931bcc0176e2b45f1cd23e16aa6f Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 19 May 2023 11:30:50 +0800 Subject: [PATCH 17/17] =?UTF-8?q?perf:=20=E6=9A=82=E5=AD=98=E4=B8=80?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/db/fields.py | 19 ++++--------------- apps/terminal/applets/chrome/app.py | 7 +++++-- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 3897a8c5e..adfe94769 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -315,8 +315,7 @@ class RelatedManager: else: queryset = to_model.objects.all() q = cls.get_filter_q(value, to_model) - print("Q: ", q) - return queryset.filter(q) + return queryset.filter(q).distinct() @staticmethod def get_ip_in_q(name, val): @@ -378,22 +377,12 @@ class RelatedManager: q = Q(**{lookup: val}) elif match == "not": q = ~Q(**{name: val}) - elif match == "m2m": + elif match in ['m2m', 'in']: if not isinstance(val, list): val = [val] - q = Q(**{"{}__in".format(name): val}) - elif match == "in" and isinstance(val, list): - if '*' not in val: - lookup = "{}__in".format(name) - q = Q(**{lookup: val}) - else: - q = Q() + q = Q() if '*' in val else Q(**{"{}__in".format(name): val}) else: - if val == '*': - q = Q() - else: - q = Q(**{name: val}) - + q = Q() if val == '*' else Q(**{name: val}) filters &= q return filters diff --git a/apps/terminal/applets/chrome/app.py b/apps/terminal/applets/chrome/app.py index ced2456c8..86a664c27 100644 --- a/apps/terminal/applets/chrome/app.py +++ b/apps/terminal/applets/chrome/app.py @@ -101,6 +101,7 @@ class StepAction: else: driver.switch_to.frame(target) + def execute_action(driver: webdriver.Chrome, step: StepAction) -> bool: try: return step.execute(driver) @@ -197,8 +198,10 @@ def default_chrome_driver_options(): # 禁用开发者工具 options.add_argument("--disable-dev-tools") # 禁用 密码管理器弹窗 - prefs = {"credentials_enable_service": False, - "profile.password_manager_enabled": False} + prefs = { + "credentials_enable_service": False, + "profile.password_manager_enabled": False + } options.add_experimental_option("prefs", prefs) options.add_experimental_option("excludeSwitches", ['enable-automation']) return options