# -*- coding: utf-8 -*- # import re import threading import time import uuid from collections import defaultdict from django.core.cache import cache from django.db import models, transaction from django.db.models import Q, Manager from django.db.transaction import atomic from django.utils.translation import ugettext from django.utils.translation import ugettext_lazy as _ from common.db.models import output_as_string from common.utils import get_logger from common.utils.lock import DistributedLock from orgs.mixins.models import OrgManager, JMSOrgBaseModel from orgs.models import Organization from orgs.utils import get_current_org, tmp_to_org, tmp_to_root_org __all__ = ['Node', 'FamilyMixin', 'compute_parent_key', 'NodeQuerySet'] logger = get_logger(__name__) def compute_parent_key(key): try: return key[:key.rindex(':')] except ValueError: return '' class NodeQuerySet(models.QuerySet): pass class FamilyMixin: __parents = None __children = None __all_children = None is_node = True child_mark: int @staticmethod def clean_children_keys(nodes_keys): sort_key = lambda k: [int(i) for i in k.split(':')] nodes_keys = sorted(list(nodes_keys), key=sort_key) nodes_keys_clean = [] base_key = '' for key in nodes_keys: if key.startswith(base_key + ':'): continue nodes_keys_clean.append(key) base_key = key return nodes_keys_clean @classmethod def get_node_all_children_key_pattern(cls, key, with_self=True): pattern = r'^{0}:'.format(key) if with_self: pattern += r'|^{0}$'.format(key) return pattern @classmethod def get_node_children_key_pattern(cls, key, with_self=True): pattern = r'^{0}:[0-9]+$'.format(key) if with_self: pattern += r'|^{0}$'.format(key) return pattern def get_children_key_pattern(self, with_self=False): return self.get_node_children_key_pattern(self.key, with_self=with_self) def get_all_children_pattern(self, with_self=False): return self.get_node_all_children_key_pattern(self.key, with_self=with_self) def is_children(self, other): children_pattern = other.get_children_key_pattern(with_self=False) return re.match(children_pattern, self.key) def get_children(self, with_self=False): q = Q(parent_key=self.key) if with_self: q |= Q(key=self.key) return Node.objects.filter(q) def get_all_children(self, with_self=False): q = Q(key__istartswith=f'{self.key}:') if with_self: q |= Q(key=self.key) return Node.objects.filter(q) @classmethod def get_ancestor_queryset(cls, queryset, with_self=True): parent_keys = set() for i in queryset: parent_keys.update(set(i.get_ancestor_keys(with_self=with_self))) queryset = queryset.model.objects.filter(key__in=list(parent_keys)).distinct() return queryset @property def children(self): return self.get_children(with_self=False) @property def all_children(self): return self.get_all_children(with_self=False) def create_child(self, value=None, _id=None): with atomic(savepoint=False): child_key = self.get_next_child_key() if value is None: value = child_key child = self.__class__.objects.create( id=_id, key=child_key, value=value ) return child def get_or_create_child(self, value, _id=None): """ :return: Node, bool (created) """ children = self.get_children() exist = children.filter(value=value).exists() if exist: child = children.filter(value=value).first() created = False else: child = self.create_child(value, _id) created = True return child, created def get_valid_child_mark(self): key = "{}:{}".format(self.key, self.child_mark) if not self.__class__.objects.filter(key=key).exists(): return self.child_mark children_keys = self.get_children().values_list('key', flat=True) children_keys_last = [key.split(':')[-1] for key in children_keys] children_keys_last = [int(k) for k in children_keys_last if k.strip().isdigit()] max_key_last = max(children_keys_last) if children_keys_last else 1 return max_key_last + 1 def get_next_child_key(self): child_mark = self.get_valid_child_mark() key = "{}:{}".format(self.key, child_mark) self.child_mark = child_mark + 1 self.save() return key def get_next_child_preset_name(self): name = ugettext("New node") values = [ child.value[child.value.rfind(' '):] for child in self.get_children() if child.value.startswith(name) ] values = [int(value) for value in values if value.strip().isdigit()] count = max(values) + 1 if values else 1 return '{} {}'.format(name, count) # Parents @classmethod def get_node_ancestor_keys(cls, key, with_self=False): parent_keys = [] key_list = key.split(":") if not with_self: key_list.pop() for i in range(len(key_list)): parent_keys.append(":".join(key_list)) key_list.pop() return parent_keys def get_ancestor_keys(self, with_self=False): return self.get_node_ancestor_keys(self.key, with_self=with_self) @property def ancestors(self): return self.get_ancestors(with_self=False) def get_ancestors(self, with_self=False): ancestor_keys = self.get_ancestor_keys(with_self=with_self) return self.__class__.objects.filter(key__in=ancestor_keys) # @property # def parent_key(self): # parent_key = ":".join(self.key.split(":")[:-1]) # return parent_key def compute_parent_key(self): return compute_parent_key(self.key) def is_parent(self, other): return other.is_children(self) @property def parent(self): if self.is_org_root(): return self parent_key = self.parent_key return Node.objects.get(key=parent_key) @parent.setter def parent(self, parent): if not self.is_node: self.key = parent.key + ':fake' return children = self.get_all_children() old_key = self.key with transaction.atomic(): self.key = parent.get_next_child_key() self.save() for child in children: child.key = child.key.replace(old_key, self.key, 1) child.save() def get_siblings(self, with_self=False): key = ':'.join(self.key.split(':')[:-1]) pattern = r'^{}:[0-9]+$'.format(key) sibling = Node.objects.filter( key__regex=pattern.format(self.key) ) if not with_self: sibling = sibling.exclude(key=self.key) return sibling @classmethod def create_node_by_full_value(cls, full_value): if not full_value: return [] nodes_family = full_value.split('/') nodes_family = [v for v in nodes_family if v] org_root = cls.org_root() if nodes_family[0] == org_root.value: nodes_family = nodes_family[1:] return cls.create_nodes_recurse(nodes_family, org_root) @classmethod def create_nodes_recurse(cls, values, parent=None): values = [v for v in values if v] if not values: return None if parent is None: parent = cls.org_root() value = values[0] child, created = parent.get_or_create_child(value=value) if len(values) == 1: return child return cls.create_nodes_recurse(values[1:], child) def get_family(self): ancestors = self.get_ancestors() children = self.get_all_children() return [*tuple(ancestors), self, *tuple(children)] class NodeAllAssetsMappingMixin: # Use a new plan # { org_id: { node_key: [ asset1_id, asset2_id ] } } orgid_nodekey_assetsid_mapping = defaultdict(dict) locks_for_get_mapping_from_cache = defaultdict(threading.Lock) @classmethod def get_lock(cls, org_id): lock = cls.locks_for_get_mapping_from_cache[str(org_id)] return lock @classmethod def get_node_all_asset_ids_mapping(cls, org_id): _mapping = cls.get_node_all_asset_ids_mapping_from_memory(org_id) if _mapping: return _mapping logger.debug(f'Get node asset mapping from memory failed, acquire thread lock: ' f'thread={threading.get_ident()} ' f'org_id={org_id}') with cls.get_lock(org_id): logger.debug(f'Acquired thread lock ok. check if mapping is in memory now: ' f'thread={threading.get_ident()} ' f'org_id={org_id}') _mapping = cls.get_node_all_asset_ids_mapping_from_memory(org_id) if _mapping: logger.debug(f'Mapping is already in memory now: ' f'thread={threading.get_ident()} ' f'org_id={org_id}') return _mapping _mapping = cls.get_node_all_asset_ids_mapping_from_cache_or_generate_to_cache(org_id) cls.set_node_all_asset_ids_mapping_to_memory(org_id, mapping=_mapping) return _mapping # from memory @classmethod def get_node_all_asset_ids_mapping_from_memory(cls, org_id): mapping = cls.orgid_nodekey_assetsid_mapping.get(org_id, {}) return mapping @classmethod def set_node_all_asset_ids_mapping_to_memory(cls, org_id, mapping): cls.orgid_nodekey_assetsid_mapping[org_id] = mapping @classmethod def expire_node_all_asset_ids_mapping_from_memory(cls, org_id): org_id = str(org_id) cls.orgid_nodekey_assetsid_mapping.pop(org_id, None) @classmethod def expire_all_orgs_node_all_asset_ids_mapping_from_memory(cls): orgs = Organization.objects.all() org_ids = [str(org.id) for org in orgs] org_ids.append(Organization.ROOT_ID) for id in org_ids: cls.expire_node_all_asset_ids_mapping_from_memory(id) # get order: from memory -> (from cache -> to generate) @classmethod def get_node_all_asset_ids_mapping_from_cache_or_generate_to_cache(cls, org_id): mapping = cls.get_node_all_asset_ids_mapping_from_cache(org_id) if mapping: return mapping lock_key = f'KEY_LOCK_GENERATE_ORG_{org_id}_NODE_ALL_ASSET_ids_MAPPING' with DistributedLock(lock_key): # 这里使用无限期锁，原因是如果这里卡住了，就卡在数据库了，说明 # 数据库繁忙，所以不应该再有线程执行这个操作，使数据库忙上加忙 _mapping = cls.get_node_all_asset_ids_mapping_from_cache(org_id) if _mapping: return _mapping _mapping = cls.generate_node_all_asset_ids_mapping(org_id) cls.set_node_all_asset_ids_mapping_to_cache(org_id=org_id, mapping=_mapping) return _mapping @classmethod def get_node_all_asset_ids_mapping_from_cache(cls, org_id): cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) mapping = cache.get(cache_key) logger.info(f'Get node asset mapping from cache {bool(mapping)}: ' f'thread={threading.get_ident()} ' f'org_id={org_id}') return mapping @classmethod def set_node_all_asset_ids_mapping_to_cache(cls, org_id, mapping): cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) cache.set(cache_key, mapping, timeout=None) @classmethod def expire_node_all_asset_ids_mapping_from_cache(cls, org_id): cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) cache.delete(cache_key) @staticmethod def _get_cache_key_for_node_all_asset_ids_mapping(org_id): return 'ASSETS_ORG_NODE_ALL_ASSET_ids_MAPPING_{}'.format(org_id) @classmethod def generate_node_all_asset_ids_mapping(cls, org_id): from .asset import Asset logger.info(f'Generate node asset mapping: ' f'thread={threading.get_ident()} ' f'org_id={org_id}') t1 = time.time() with tmp_to_org(org_id): node_ids_key = Node.objects.annotate( char_id=output_as_string('id') ).values_list('char_id', 'key') # * 直接取出全部. filter(node__org_id=org_id)(大规模下会更慢) nodes_asset_ids = Asset.nodes.through.objects.all() \ .annotate(char_node_id=output_as_string('node_id')) \ .annotate(char_asset_id=output_as_string('asset_id')) \ .values_list('char_node_id', 'char_asset_id') node_id_ancestor_keys_mapping = { node_id: cls.get_node_ancestor_keys(node_key, with_self=True) for node_id, node_key in node_ids_key } nodeid_assetsid_mapping = defaultdict(set) for node_id, asset_id in nodes_asset_ids: nodeid_assetsid_mapping[node_id].add(asset_id) t2 = time.time() mapping = defaultdict(set) for node_id, node_key in node_ids_key: asset_ids = nodeid_assetsid_mapping[node_id] node_ancestor_keys = node_id_ancestor_keys_mapping[node_id] for ancestor_key in node_ancestor_keys: mapping[ancestor_key].update(asset_ids) t3 = time.time() logger.info('t1-t2(DB Query): {} s, t3-t2(Generate mapping): {} s'.format(t2 - t1, t3 - t2)) return mapping class NodeAssetsMixin(NodeAllAssetsMappingMixin): org_id: str key = '' id = None objects: Manager def get_all_assets(self): from .asset import Asset q = Q(nodes__key__startswith=f'{self.key}:') | Q(nodes__key=self.key) return Asset.objects.filter(q).distinct() @classmethod def get_node_all_assets_by_key_v2(cls, key): # 最初的写法是： # Asset.objects.filter(Q(nodes__key__startswith=f'{node.key}:') | Q(nodes__id=node.id)) # 可是 startswith 会导致表关联时 Asset 索引失效 from .asset import Asset node_ids = cls.objects.filter( Q(key__startswith=f'{key}:') | Q(key=key) ).values_list('id', flat=True).distinct() assets = Asset.objects.filter( nodes__id__in=list(node_ids) ).distinct() return assets def get_assets(self): from .asset import Asset assets = Asset.objects.filter(nodes=self) return assets.distinct() def get_assets_for_tree(self): return self.get_assets().only( "id", "name", "address", "platform_id", "org_id", "is_active" ).prefetch_related('platform') def get_all_assets_for_tree(self): return self.get_all_assets().only( "id", "name", "address", "platform_id", "org_id", "is_active" ).prefetch_related('platform') def get_valid_assets(self): return self.get_assets().valid() def get_all_valid_assets(self): return self.get_all_assets().valid() @classmethod def get_nodes_all_asset_ids_by_keys(cls, nodes_keys): nodes = Node.objects.filter(key__in=nodes_keys) asset_ids = cls.get_nodes_all_assets(*nodes).values_list('id', flat=True) return asset_ids @classmethod def get_nodes_all_assets(cls, *nodes): from .asset import Asset node_ids = set() descendant_node_query = Q() for n in nodes: node_ids.add(n.id) descendant_node_query |= Q(key__istartswith=f'{n.key}:') if descendant_node_query: _ids = Node.objects.order_by().filter(descendant_node_query).values_list('id', flat=True) node_ids.update(_ids) return Asset.objects.order_by().filter(nodes__id__in=node_ids).distinct() def get_all_asset_ids(self): asset_ids = self.get_all_asset_ids_by_node_key(org_id=self.org_id, node_key=self.key) return set(asset_ids) @classmethod def get_all_asset_ids_by_node_key(cls, org_id, node_key): org_id = str(org_id) nodekey_assetsid_mapping = cls.get_node_all_asset_ids_mapping(org_id) asset_ids = nodekey_assetsid_mapping.get(node_key, []) return set(asset_ids) class SomeNodesMixin: key = '' default_key = '1' empty_key = '-11' empty_value = _("empty") def is_default_node(self): return self.key == self.default_key def is_org_root(self): if self.key.isdigit(): return True else: return False @classmethod def org_root(cls): # 如果使用current_org 在set_current_org时会死循环 ori_org = get_current_org() if ori_org and ori_org.is_default(): return cls.default_node() if ori_org and ori_org.is_root(): return None org_roots = cls.org_root_nodes() org_roots_length = len(org_roots) if org_roots_length == 1: root = org_roots[0] return root elif org_roots_length == 0: root = cls.create_org_root_node() return root else: error = 'Current org {} root node not 1, get {}'.format(ori_org, org_roots_length) raise ValueError(error) @classmethod def default_node(cls): default_org = Organization.default() with tmp_to_org(default_org): defaults = {'value': default_org.name} obj, created = cls.objects.get_or_create(defaults=defaults, key=cls.default_key) return obj @classmethod def create_org_root_node(cls): ori_org = get_current_org() with transaction.atomic(): key = cls.get_next_org_root_node_key() root = cls.objects.create(key=key, value=ori_org.name) return root @classmethod def get_next_org_root_node_key(cls): with tmp_to_root_org(): org_nodes_roots = cls.org_root_nodes() org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) if not org_nodes_roots_keys: org_nodes_roots_keys = ['1'] max_key = max([int(k) for k in org_nodes_roots_keys]) key = str(max_key + 1) if max_key > 0 else '2' return key @classmethod def org_root_nodes(cls): root_nodes = cls.objects.filter(parent_key='', key__regex=r'^[0-9]+$') \ .exclude(key__startswith='-').order_by('key') return root_nodes class Node(JMSOrgBaseModel, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) key = models.CharField(unique=True, max_length=64, verbose_name=_("Key")) # '1:1:1:1' value = models.CharField(max_length=128, verbose_name=_("Value")) full_value = models.CharField(max_length=4096, verbose_name=_('Full value'), default='') child_mark = models.IntegerField(default=0) date_create = models.DateTimeField(auto_now_add=True) parent_key = models.CharField( max_length=64, verbose_name=_("Parent key"), db_index=True, default='' ) assets_amount = models.IntegerField(default=0) objects = OrgManager.from_queryset(NodeQuerySet)() is_node = True _parents = None class Meta: verbose_name = _("Node") ordering = ['parent_key', 'value'] permissions = [ ('match_node', _('Can match node')), ] def __str__(self): return self.full_value # def __eq__(self, other): # if not other: # return False # return self.id == other.id # def __gt__(self, other): self_key = [int(k) for k in self.key.split(':')] other_key = [int(k) for k in other.key.split(':')] self_parent_key = self_key[:-1] other_parent_key = other_key[:-1] if self_parent_key and self_parent_key == other_parent_key: return self.value > other.value return self_key > other_key def __lt__(self, other): return not self.__gt__(other) @property def name(self): return self.value def computed_full_value(self): # 不要在列表中调用该属性 values = self.__class__.objects.filter( key__in=self.get_ancestor_keys() ).values_list('key', 'value') values = [v for k, v in sorted(values, key=lambda x: len(x[0]))] values.append(str(self.value)) return '/' + '/'.join(values) @property def level(self): return len(self.key.split(':')) def as_tree_node(self): from common.tree import TreeNode name = '{} ({})'.format(self.value, self.assets_amount) data = { 'id': self.key, 'name': name, 'title': name, 'pId': self.parent_key, 'isParent': True, 'open': self.is_org_root(), 'meta': { 'data': { "id": self.id, "name": self.name, "value": self.value, "key": self.key, "assets_amount": self.assets_amount, }, 'type': 'node' } } tree_node = TreeNode(**data) return tree_node def has_offspring_assets(self): # 拥有后代资产 return self.get_all_assets().exists() def delete(self, using=None, keep_parents=False): if self.has_offspring_assets(): return self.all_children.delete() return super().delete(using=using, keep_parents=keep_parents) def update_child_full_value(self): nodes = self.get_all_children(with_self=True) sort_key_func = lambda n: [int(i) for i in n.key.split(':')] nodes_sorted = sorted(list(nodes), key=sort_key_func) nodes_mapper = {n.key: n for n in nodes_sorted} if not self.is_org_root(): # 如果是org_root，那么parent_key为'', parent为自己，所以这种情况不处理 # 更新自己时，自己的parent_key获取不到 nodes_mapper.update({self.parent_key: self.parent}) for node in nodes_sorted: parent = nodes_mapper.get(node.parent_key) if not parent: if node.parent_key: logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}') continue node.full_value = parent.full_value + '/' + node.value self.__class__.objects.bulk_update(nodes, ['full_value']) def save(self, *args, **kwargs): self.full_value = self.computed_full_value() instance = super().save(*args, **kwargs) self.update_child_full_value() return instance