pull/2874/head
ibuler 2019-07-01 18:22:40 +08:00
parent 8f699fa366
commit ae690050e7
12 changed files with 234 additions and 203 deletions

View File

@ -50,19 +50,14 @@ class AssetSystemUserSerializer(serializers.ModelSerializer):
"""
查看授权的资产系统用户的数据结构这个和AssetSerializer不同字段少
"""
actions = serializers.SerializerMethodField()
class Meta:
model = SystemUser
fields = (
'id', 'name', 'username', 'priority',
'protocol', 'comment', 'login_mode', 'actions',
'protocol', 'comment', 'login_mode',
)
@staticmethod
def get_actions(obj):
return [action.name for action in obj.actions]
class SystemUserSimpleSerializer(serializers.ModelSerializer):
"""

View File

@ -54,17 +54,19 @@ class NodeUtil:
def sorted_by(node):
return [int(i) for i in node.key.split(':')]
def get_all_nodes(self):
def get_queryset(self):
all_nodes = Node.objects.all()
if self.with_assets_amount:
now = time.time()
all_nodes = all_nodes.prefetch_related(
Prefetch('assets', queryset=Asset.objects.all().only('id'))
)
all_nodes = list(all_nodes)
for node in all_nodes:
node._assets = set(node.assets.all())
all_nodes = sorted(all_nodes, key=self.sorted_by)
return all_nodes
def get_all_nodes(self):
all_nodes = sorted(self.get_queryset(), key=self.sorted_by)
guarder = Node(key='', value='Guarder')
guarder._assets = []
@ -119,11 +121,11 @@ class NodeUtil:
def get_nodes_by_queryset(self, queryset):
nodes = []
for n in queryset:
node = self._nodes.get(n.key)
node = self.get_node_by_key(n.key)
if not node:
continue
nodes.append(nodes)
return [self]
nodes.append(node)
return nodes
def get_node_by_key(self, key):
return self._nodes.get(key)
@ -156,11 +158,17 @@ class NodeUtil:
tree_nodes.add(node)
if with_children:
tree_nodes.update(node._children)
for n in tree_nodes:
delattr(n, '_children')
delattr(n, '_parents')
return list(tree_nodes)
def get_nodes_parents(self, nodes, with_self=True):
parents = set()
for n in nodes:
node = self.get_node_by_key(n.key)
parents.update(set(node._parents))
if with_self:
parents.add(node)
return parents
def test_node_tree():
tree = NodeUtil()

View File

@ -35,7 +35,8 @@ class AssetPermissionViewSet(viewsets.ModelViewSet):
permission_classes = (IsOrgAdmin,)
def get_serializer_class(self):
if self.action in ("list", 'retrieve'):
if self.action in ("list", 'retrieve') and \
self.request.query_params.get("display"):
return serializers.AssetPermissionListSerializer
return self.serializer_class

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
import time
from hashlib import md5
from django.core.cache import cache
from django.conf import settings
@ -261,14 +261,19 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView)
nodes = util.get_nodes_with_assets()
print("22222222222222")
for node, assets in nodes.items():
now = time.time()
print("Parse to node")
data = parse_node_to_tree_node(node)
print("parse to node end, using: {0:.2f}".format(time.time() - now))
queryset.append(data)
if not self.show_assets:
continue
for asset, system_users in assets.items():
now1 = time.time()
print("parse to asset")
data = parse_asset_to_tree_node(node, asset, system_users)
print("parse to asset end, using: {0:.2f}".format(time.time()-now1))
queryset.append(data)
queryset = sorted(queryset)
return queryset

View File

@ -74,13 +74,13 @@ class AssetPermissionForm(OrgModelForm):
'system_users': forms.SelectMultiple(
attrs={'class': 'select2', 'data-placeholder': _('System user')}
),
'action': forms.CheckboxSelectMultiple()
'actions': forms.CheckboxSelectMultiple()
}
labels = {
'nodes': _("Node"),
}
help_texts = {
'action': _('Tips: The RDP protocol does not support separate '
'actions': _('Tips: The RDP protocol does not support separate '
'controls for uploading or downloading files')
}

View File

@ -36,7 +36,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='assetpermission',
name='action',
field=models.IntegerField(choices=[(255, 'All'), (1, 'Connect'), (2, 'Upload file'), (5, 'Upload download'), (4, 'Download file')], default=255, verbose_name='Action'),
field=models.IntegerField(choices=[(255, 'All'), (1, 'Connect'), (2, 'Upload file'), (4, 'Download file'), (6, 'Upload download')], default=255, verbose_name='Actions'),
),
migrations.RunPython(migrate_old_actions),
]

View File

@ -14,4 +14,9 @@ class Migration(migrations.Migration):
model_name='assetpermission',
name='actions',
),
migrations.RenameField(
model_name='assetpermission',
old_name='action',
new_name='actions',
),
]

View File

@ -39,39 +39,46 @@ class ActionFlag:
UPLOAD = 0b00000010
DOWNLOAD = 0b00000100
UPDOWNLOAD = UPLOAD | DOWNLOAD
CONNECT_UPLOADOWN = CONNECT | UPDOWNLOAD
ALL = 0b11111111
NAME_MAP = {
"connect": CONNECT,
"upload": UPLOAD,
"download": DOWNLOAD,
"updownload": UPDOWNLOAD,
"all": ALL,
}
CHOICES = (
DB_CHOICES = (
(ALL, _('All')),
(CONNECT, _('Connect')),
(UPDOWNLOAD, _("Upload download")),
(UPLOAD, _('Upload file')),
(DOWNLOAD, _('Download file')),
(UPDOWNLOAD, _("Upload download")),
)
NAME_MAP = {
ALL: "all",
CONNECT: "connect",
UPLOAD: "upload_file",
DOWNLOAD: "download_file",
UPDOWNLOAD: "updownload",
}
NAME_MAP_REVERSE = dict({v: k for k, v in NAME_MAP.items()})
CHOICES = []
for i, j in DB_CHOICES:
CHOICES.append((NAME_MAP[i], j))
@classmethod
def value_to_choices(cls, value):
value = int(value)
if value == cls.ALL:
return [cls.ALL]
elif value == cls.UPDOWNLOAD:
return [cls.UPDOWNLOAD]
elif value == cls.CONNECT_UPLOADOWN:
return [cls.CONNECT, cls.UPDOWNLOAD]
else:
return [i for i in dict(cls.CHOICES) if i == i & int(value)]
choices = [cls.NAME_MAP[i] for i, j in cls.DB_CHOICES if value & i == i]
return choices
@classmethod
def choices_to_value(cls, value):
return reduce(lambda x, y: int(x) | int(y), value)
def to_choices(x, y):
x = cls.NAME_MAP_REVERSE.get(x, 0)
y = cls.NAME_MAP_REVERSE.get(y, 0)
return x | y
return reduce(to_choices, value)
@classmethod
def choices(cls):
return [(cls.NAME_MAP[i], j) for i, j in cls.DB_CHOICES]
class AssetPermission(BasePermission):
@ -79,7 +86,7 @@ class AssetPermission(BasePermission):
nodes = models.ManyToManyField('assets.Node', related_name='granted_by_permissions', blank=True, verbose_name=_("Nodes"))
system_users = models.ManyToManyField('assets.SystemUser', related_name='granted_by_permissions', verbose_name=_("System user"))
# actions = models.ManyToManyField(Action, related_name='permissions', blank=True, verbose_name=_('Action'))
action = models.IntegerField(choices=ActionFlag.CHOICES, default=ActionFlag.ALL, verbose_name=_("Action"))
actions = models.IntegerField(choices=ActionFlag.DB_CHOICES, default=ActionFlag.ALL, verbose_name=_("Actions"))
class Meta:
unique_together = [('org_id', 'name')]

View File

@ -38,7 +38,7 @@ class ActionDisplayField(ActionField):
class AssetPermissionCreateUpdateSerializer(BulkOrgResourceModelSerializer):
action = ActionField()
actions = ActionField()
class Meta:
model = AssetPermission
@ -51,7 +51,7 @@ class AssetPermissionListSerializer(BulkOrgResourceModelSerializer):
assets = StringManyToManyField(many=True, read_only=True)
nodes = StringManyToManyField(many=True, read_only=True)
system_users = StringManyToManyField(many=True, read_only=True)
action = ActionDisplayField()
actions = ActionDisplayField()
is_valid = serializers.BooleanField()
is_expired = serializers.BooleanField()

View File

@ -122,8 +122,8 @@ function format(d) {
if (d.system_users.length > 0) {
data += makeLabel(["{% trans 'System user' %}", d.system_users.join(", ")])
}
if (d.action.length > 0) {
data += makeLabel(["{% trans 'Action' %}", d.action.join(", ")])
if (d.actions.length > 0) {
data += makeLabel(["{% trans 'Action' %}", d.actions.join(", ")])
}
return data
}
@ -180,7 +180,7 @@ function initTable() {
$(td).html(update_btn + del_btn);
}}
],
ajax_url: '{% url "api-perms:asset-permission-list" %}',
ajax_url: '{% url "api-perms:asset-permission-list" %}?display=1',
columns: [
{data: "id"}, {data: "name"}, {data: "users"},
{data: "user_groups"}, {data: "assets"},

View File

@ -17,8 +17,8 @@ from orgs.utils import set_to_root_org
from common.utils import get_logger
from common.tree import TreeNode
from .. import const
from ..models import AssetPermission, Action
from ..hands import Node
from ..models import AssetPermission, Action, ActionFlag
from ..hands import Node, Asset
from assets.utils import NodeUtil
logger = get_logger(__file__)
@ -31,17 +31,57 @@ __all__ = [
]
class TreeNodeCounter(NodeUtil):
def __init__(self, nodes):
self.__nodes = nodes
super().__init__(with_assets_amount=True)
def get_queryset(self):
return self.__nodes
def timeit(func):
def wrapper(*args, **kwargs):
logger.debug("Start call: {}".format(func.__name__))
now = time.time()
result = func(*args, **kwargs)
using = time.time() - now
logger.debug("Call {} end, using: {:.2}s".format(func.__name__, using))
return result
return wrapper
class GenerateTree:
def __init__(self):
"""
nodes: {"node_instance": {
"asset_instance": set("system_user")
nodes = {
"<node1>": {
"system_users": {
"system_user": action,
"system_user2": action,
},
"assets": set([<asset_instance>]),
}
}
assets = {
"<asset_instance2>": {
"system_user": action,
"system_user2": action,
},
}
"""
self.node_util = NodeUtil()
self.nodes = defaultdict(dict)
self._node_util = None
self.nodes = defaultdict(lambda: {"system_users": defaultdict(int), "assets": set(), "assets_amount": 0})
self.assets = defaultdict(lambda: defaultdict(int))
self._root_node = None
self._ungroup_node = None
self._nodes_with_assets = None
@property
def node_util(self):
if not self._node_util:
self._node_util = NodeUtil()
return self._node_util
@property
def root_node(self):
@ -66,35 +106,79 @@ class GenerateTree:
node_key = '0:0'
node_value = _("Default")
node = Node(id=node_id, key=node_key, value=node_value)
self.add_node(node)
self.add_node(node, {})
self._ungroup_node = node
return node
def add_asset(self, asset, system_users):
@timeit
def add_assets_without_system_users(self, assets):
for asset in assets:
self.add_asset(asset, {})
@timeit
def add_assets(self, assets):
for asset, system_users in assets.items():
self.add_asset(asset, system_users)
@timeit
def add_asset(self, asset, system_users=None):
nodes = asset.nodes.all()
for node in nodes:
if node in self.nodes:
self.nodes[node][asset].update(system_users)
else:
self.nodes[self.ungrouped_node][asset].update(system_users)
nodes = self.node_util.get_nodes_by_queryset(nodes)
if not system_users:
system_users = defaultdict(int)
else:
system_users = {k: v for k, v in system_users.items()}
_system_users = self.assets[asset]
for system_user, action in _system_users.items():
system_users[system_user] |= action
def get_nodes(self):
for node in self.nodes:
assets = set(self.nodes.get(node).keys())
for n in self.nodes.keys():
if n.key.startswith(node.key + ':'):
assets.update(set(self.nodes[n].keys()))
node.assets_amount = len(assets)
return self.nodes
# 获取父节点们
parents = self.node_util.get_nodes_parents(nodes, with_self=True)
for node in parents:
_system_users = self.nodes[node]["system_users"]
self.nodes[node]["assets_amount"] += 1
for system_user, action in _system_users.items():
system_users[system_user] |= action
def add_node(self, node):
self.nodes[node] = defaultdict(set)
# 过滤系统用户的协议
system_users = {s: v for s, v in system_users.items() if asset.has_protocol(s.protocol)}
self.assets[asset] = system_users
in_nodes = set(self.nodes.keys()) & set(nodes)
if not in_nodes:
self.nodes[self.ungrouped_node]["assets_amount"] += 1
self.nodes[self.ungrouped_node]["assets"].add(system_users)
return
for node in in_nodes:
self.nodes[node]["assets"].add(asset)
def add_node(self, node, system_users=None):
if not system_users:
system_users = defaultdict(int)
self.nodes[node]["system_users"] = system_users
# 添加树节点
@timeit
def add_nodes(self, nodes):
need_nodes = self.node_util.get_family(nodes, with_children=True)
for node in need_nodes:
self.add_node(node)
_nodes = nodes.keys()
family = self.node_util.get_family(_nodes, with_children=True)
for node in family:
self.add_node(node, nodes.get(node, {}))
def get_assets(self):
return dict(self.assets)
@timeit
def get_nodes_with_assets(self):
if self._nodes_with_assets:
return self._nodes_with_assets
nodes = {}
for node, values in self.nodes.items():
node._assets_amount = values["assets_amount"]
nodes[node] = {asset: self.assets.get(asset, {}) for asset in values["assets"]}
self._nodes_with_assets = nodes
return dict(nodes)
def get_user_permissions(user, include_group=True):
@ -131,17 +215,6 @@ def get_system_user_permissions(system_user):
)
def timeit(func):
def wrapper(*args, **kwargs):
logger.debug("Start call: {}".format(func.__name__))
now = time.time()
result = func(*args, **kwargs)
using = time.time() - now
logger.debug("Call {} end, using: {:.2}".format(func.__name__, using))
return result
return wrapper
class AssetPermissionCacheMixin:
CACHE_KEY_PREFIX = '_ASSET_PERM_CACHE_'
CACHE_META_KEY_PREFIX = '_ASSET_PERM_META_KEY_'
@ -216,6 +289,16 @@ class AssetPermissionCacheMixin:
cached = cache.get(self.system_key)
return cached
def get_assets(self):
if self._is_not_using_cache():
return self.get_assets_from_cache()
elif self._is_refresh_cache():
self.expire_cache()
return self.get_assets_from_cache()
else:
self.expire_cache()
return self.get_assets_without_cache()
def get_system_users(self):
if self._is_using_cache():
return self.get_system_user_from_cache()
@ -282,57 +365,6 @@ class AssetPermissionCacheMixin:
cache.delete_pattern(key)
class FlatPermissionQueryset(set):
def add_many(self, assets_or_nodes, system_users, action, rtp="asset"):
print("Add many: {}-{}-{}".format(len(assets_or_nodes), len(system_users), action))
if not any([assets_or_nodes, system_users, action]):
return
iterable = itertools.product(assets_or_nodes, system_users, [action])
for source, sysuser, action in iterable:
permission = FlatPermission(source, sysuser, action, rtp=rtp)
print("ADDDDDDDDDDDDDDDd")
self.add(permission)
def group_by_resource(self):
resources = defaultdict(lambda: defaultdict(int))
for i in self:
resources[i.resource][i.system_user] |= i.action
return resources
class FlatPermission:
def __init__(self, assets_or_node, system_user, action, rtp="asset"):
self.id = "{}_{}_{}".format(assets_or_node.id, system_user.id, action)
self.resource = assets_or_node
self.resource_type = rtp
self.system_user = system_user
self.action = action
def __eq__(self, other):
if self.id == other.id:
return True
# 资产不同
if self.resource_type == "asset" and self.id != other.id:
return False
# 不是子节点
elif self.resource_type == "node" and not other.resource.key.startswith(self.resource.key):
return False
# 系统用户优先级大于后者,则相同
if self.system_user.priority > self.system_user.priority:
return True
# 如果系统用户不同,则不同
elif self.system_user != other.system_user:
return False
# 如果action为与后的结果则相同
if self.action == self.action | other.action:
return True
return False
def __hash__(self):
return hash(self.id)
class AssetPermissionUtil(AssetPermissionCacheMixin):
get_permissions_map = {
"User": get_user_permissions,
@ -353,6 +385,7 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
self.tree = GenerateTree()
self.change_org_if_need()
self.nodes = None
self._nodes = None
@staticmethod
def change_org_if_need():
@ -380,29 +413,32 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
返回用户/组授权规则直接关联的节点
:return: {node1: {system_user1: {'actions': set()},}}
"""
queryset = FlatPermissionQueryset()
nodes = defaultdict(lambda: defaultdict(int))
for perm in self.permissions:
actions = perm.action
actions = [perm.actions]
system_users = perm.system_users.all()
nodes = perm.nodes.all()
queryset.add_many(nodes, system_users, actions, rtp="nodes")
print(queryset)
return queryset.group_by_resource()
_nodes = perm.nodes.all()
for node, system_user, action in itertools.product(_nodes, system_users, actions):
nodes[node][system_user] |= action
self.tree.add_nodes(nodes)
return nodes
@timeit
def get_assets_direct(self):
"""
返回用户授权规则直接关联的资产
:return: {asset1: {system_user1: {'actions': set()},}}
:return: {asset1: {system_user1: 1,}}
"""
queryset = FlatPermissionQueryset()
assets = defaultdict(lambda: defaultdict(int))
for perm in self.permissions:
action = perm.action
assets = perm.assets.all()
actions = [perm.actions]
_assets = perm.assets.all().prefetch_related('nodes', 'protocols')
system_users = perm.system_users.all()
queryset.add_many(assets, system_users, action, rtp="assets")
print(queryset)
return queryset.group_by_resource()
iterable = itertools.product(_assets, system_users, actions)
for asset, system_user, action in iterable:
assets[asset][system_user] |= action
self.tree.add_assets(assets)
return assets
@timeit
def get_assets_without_cache(self):
@ -411,24 +447,34 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
"""
if self._assets:
return self._assets
assets = self.get_assets_direct()
nodes = self.get_nodes_direct()
print("++++++++++++++++++++++")
print(assets)
print("---------------------")
print(nodes)
pattern = set()
for node in nodes:
pattern.add(r'^{0}$|^{0}:'.format(node.key))
pattern = '|'.join(list(pattern))
now = time.time()
print("Get node assets start")
if pattern:
assets = Asset.objects.filter(nodes__key__regex=pattern)\
.prefetch_related('nodes', "protocols").only('id', 'hostname', 'ip').distinct()
else:
assets = []
assets = list(assets)
print("Get node assets end, using: {}".format(time.time() - now))
self.tree.add_assets_without_system_users(assets)
assets = self.tree.get_assets()
self._assets = assets
return assets
@timeit
def get_nodes_with_assets_without_cache(self):
"""
返回节点并且包含资产
{"node": {"assets": set("system_user")}}
{"node": {"asset": {"system_user": 1})}}
:return:
"""
assets = self.get_assets_without_cache()
for asset, system_users in assets.items():
self.tree.add_asset(asset, system_users)
return self.tree.get_nodes()
self.get_assets_without_cache()
return self.tree.get_nodes_with_assets()
def get_system_user_without_cache(self):
system_users = set()
@ -460,9 +506,7 @@ def sort_assets(assets, order_by='hostname', reverse=False):
def parse_node_to_tree_node(node):
from .. import serializers
name = '{} ({})'.format(node.value, node.assets_amount)
node_serializer = serializers.GrantedNodeSerializer(node)
data = {
'id': node.key,
'name': name,
@ -471,7 +515,11 @@ def parse_node_to_tree_node(node):
'isParent': True,
'open': node.is_root(),
'meta': {
'node': node_serializer.data,
'node': {
"id": node.id,
"key": node.key,
"value": node.value,
},
'type': 'node'
}
}
@ -480,23 +528,21 @@ def parse_node_to_tree_node(node):
def parse_asset_to_tree_node(node, asset, system_users):
system_users_protocol_matched = [s for s in system_users if asset.has_protocol(s.protocol)]
icon_skin = 'file'
if asset.platform.lower() == 'windows':
icon_skin = 'windows'
elif asset.platform.lower() == 'linux':
icon_skin = 'linux'
system_users = []
for system_user in system_users_protocol_matched:
system_users.append({
_system_users = []
for system_user, action in system_users.items():
_system_users.append({
'id': system_user.id,
'name': system_user.name,
'username': system_user.username,
'protocol': system_user.protocol,
'priority': system_user.priority,
'login_mode': system_user.login_mode,
'actions': [action.name for action in system_user.actions],
'comment': system_user.comment,
'actions': [ActionFlag.value_to_choices(action)],
})
data = {
'id': str(asset.id),
@ -507,7 +553,7 @@ def parse_asset_to_tree_node(node, asset, system_users):
'open': False,
'iconSkin': icon_skin,
'meta': {
'system_users': system_users,
'system_users': _system_users,
'type': 'asset',
'asset': {
'id': asset.id,

View File

@ -1,36 +0,0 @@
# -*- coding: utf-8 -*-
#
from django.test import TestCase
from assets.models import Node, SystemUser
from .asset_permission import FlatPermission
from ..models import ActionFlag
class TestFlatPermissionEqual(TestCase):
def setUp(self):
node1 = Node(value="parent", key="1:1")
node2 = Node(value="child", key="1:1:1")
system_user1 = SystemUser(username="name1", name="name1", priority=20)
system_user2 = SystemUser(username="name2", name="name2", priority=10)
action1 = ActionFlag.ALL
action2 = ActionFlag.CONNECT
action3 = ActionFlag.UPDOWNLOAD
perm1 = FlatPermission(node1, system_user1, action1)
perm2 = FlatPermission(node2, system_user1, action1)
perm3 = FlatPermission(node2, system_user2, action1)
self.groups = (
(perm1, perm2, True),
(perm1, perm3, True),
)
def test_equal(self):
for k, k2, wanted in self.groups:
if (k == k2) != wanted:
print("Not equal {} {}", k, k2)