github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

[Update] 准备优化 asset user

pull/2882/head
ibuler 2019-07-04 15:36:57 +08:00
parent cdbdc853ea
commit 1448d23ca6
23 changed files with 185 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/assets/api/asset.py
View File

@ -20,6 +20,7 @@ from common.mixins import IDInCacheFilterMixin, ApiMessageMixin
from common.utils import get_logger, get_object_or_none
from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
from orgs.mixins import OrgBulkModelViewSet
from ..const import CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX
from ..models import Asset, AdminUser, Node
from .. import serializers
@ -36,7 +37,7 @@ __all__ = [
]
class AssetViewSet(IDInCacheFilterMixin, LabelFilter, ApiMessageMixin, BulkModelViewSet):
class AssetViewSet(LabelFilter, ApiMessageMixin, OrgBulkModelViewSet):
"""
API endpoint that allows Asset to be viewed or edited.
"""
@ -100,11 +101,6 @@ class AssetViewSet(IDInCacheFilterMixin, LabelFilter, ApiMessageMixin, BulkModel
queryset = self.filter_admin_user_id(queryset)
return queryset
def get_queryset(self):
queryset = super().get_queryset().distinct()
queryset = self.get_serializer_class().setup_eager_loading(queryset)
return queryset
class AssetListUpdateApi(IDInCacheFilterMixin, ListBulkCreateUpdateDestroyAPIView):
"""

11
apps/assets/api/asset_user.py
View File

@ -127,15 +127,14 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView):
return Response(serializer.data, status=status_code)
def get_object(self):
username = self.request.GET.get('username')
asset_id = self.request.GET.get('asset_id')
prefer = self.request.GET.get("prefer")
query_params = self.request.query_params
username = query_params.get('username')
asset_id = query_params.get('asset_id')
prefer = query_params.get("prefer")
asset = get_object_or_none(Asset, pk=asset_id)
try:
manger = AssetUserManager()
if prefer:
manger.prefer(prefer)
instance = manger.get(username, asset)
instance = manger.get(username, asset, prefer=prefer)
except Exception as e:
logger.error(e, exc_info=True)
return None

3
apps/assets/api/system_user.py
View File

@ -22,6 +22,7 @@ from rest_framework.pagination import LimitOffsetPagination
from common.utils import get_logger
from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
from common.mixins import IDInCacheFilterMixin
from orgs.mixins import OrgBulkModelViewSet
from ..models import SystemUser, Asset
from .. import serializers
from ..tasks import push_system_user_to_assets_manual, \
@ -39,7 +40,7 @@ __all__ = [
]
class SystemUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
class SystemUserViewSet(OrgBulkModelViewSet):
"""
System user api set, for add,delete,update,list,retrieve resource
"""

5
apps/assets/backends/asset_user.py
View File

@ -14,6 +14,11 @@ class AssetUserBackend(BaseBackend):
@classmethod
def filter(cls, username=None, assets=None, **kwargs):
queryset = cls.model.objects.all()
prefer_id = kwargs.get('prefer_id')
if prefer_id:
queryset = queryset.filter(id=prefer_id)
instances = cls.construct_authbook_objects(queryset, assets)
return instances
if username:
queryset = queryset.filter(username=username)
if assets:

4
apps/assets/backends/base.py
View File

@ -7,11 +7,13 @@ from abc import abstractmethod
class BaseBackend:
@classmethod
@abstractmethod
def filter(cls, username=None, assets=None, latest=True):
def filter(cls, username=None, assets=None, latest=True, prefer=None, prefer_id=None):
"""
:param username: 用户名
:param assets: <Asset>对象
:param latest: 是否是最新记录
:param prefer: 优先使用
:param prefer_id: 使用id
:return: 元素为<AuthBook>的可迭代对象(<list> or <QuerySet>)
"""
pass

2
apps/assets/backends/db.py
View File

@ -7,7 +7,7 @@ from .base import BaseBackend
class AuthBookBackend(BaseBackend):
@classmethod
def filter(cls, username=None, assets=None, latest=True):
def filter(cls, username=None, assets=None, latest=True, **kwargs):
queryset = AuthBook.objects.all()
if username is not None:
queryset = queryset.filter(username=username)

26
apps/assets/backends/manager.py
View File

@ -30,24 +30,22 @@ class AssetUserManager:
)
_prefer = "system_user"
_using = None
def filter(self, username=None, assets=None, latest=True):
def filter(self, username=None, assets=None, latest=True, prefer=None, prefer_id=None):
if assets is not None and not assets:
return AssetUserQuerySet([])
if self._using:
backend = dict(self.backends).get(self._using)
if not backend:
return self.none()
instances = backend.filter(username=username, assets=assets, latest=latest)
return AssetUserQuerySet(instances)
if prefer:
self._prefer = prefer
instances_map = {}
instances = []
for name, backend in self.backends:
if name != "db" and self._prefer != name:
continue
_instances = backend.filter(
username=username, assets=assets, latest=latest
username=username, assets=assets, latest=latest,
prefer=self._prefer, prefer_id=prefer_id,
)
instances_map[name] = _instances
@ -64,12 +62,12 @@ class AssetUserManager:
else:
ordering.extend(["admin_user", "system_user"])
# 根据prefer决定优先使用系统用户或管理用户谁的
ordering_instances = [instances_map.get(i) for i in ordering]
ordering_instances = [instances_map.get(i, []) for i in ordering]
instances = self._merge_instances(*ordering_instances)
return AssetUserQuerySet(instances)
def get(self, username, asset):
instances = self.filter(username, assets=[asset])
def get(self, username, asset, **kwargs):
instances = self.filter(username, assets=[asset], **kwargs)
if len(instances) == 1:
return instances[0]
elif len(instances) == 0:
@ -95,10 +93,6 @@ class AssetUserManager:
self._prefer = s
return self
def using(self, s):
self._using = s
return self
@staticmethod
def none():
return AssetUserQuerySet()

51
apps/assets/models/base.py
View File

@ -5,6 +5,7 @@ import uuid
from hashlib import md5
import sshpubkeys
from django.core.cache import cache
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
@ -34,7 +35,10 @@ class AssetUser(OrgModelMixin):
date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_ASSET_CONNECTIVITY_{}"
CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_ASSET_CONNECTIVITY"
CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_CONNECTIVITY_AMOUNT"
ASSETS_AMOUNT_CACHE_KEY = "ASSET_USER_{}_ASSETS_AMOUNT"
ASSET_USER_CACHE_TIME = 3600 * 24
_prefer = "system_user"
@ -67,6 +71,11 @@ class AssetUser(OrgModelMixin):
pass
return None
@property
def part_id(self):
i = '-'.join(str(self.id).split('-')[:3])
return i
def get_related_assets(self):
assets = self.assets.all()
return assets
@ -97,10 +106,14 @@ class AssetUser(OrgModelMixin):
self.set_asset_connectivity(asset, Connectivity.reachable())
else:
self.set_asset_connectivity(asset, Connectivity.unknown())
cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
cache.delete(cache_key)
@property
def connectivity(self):
assets = self.get_related_assets()
assets = self.get_related_assets()\
.select_related('admin_user')\
.only('id', 'hostname', 'admin_user')
data = {
'unreachable': [],
'reachable': [],
@ -118,11 +131,25 @@ class AssetUser(OrgModelMixin):
@property
def connectivity_amount(self):
return {k: len(v) for k, v in self.connectivity.items()}
cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
amount = cache.get(cache_key)
if not amount:
connectivity = {k: len(v) for k, v in self.connectivity.items()}
cache.set(cache_key, connectivity, self.ASSET_USER_CACHE_TIME)
return amount
@property
def assets_amount(self):
return self.get_related_assets().count()
cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id)
cached = cache.get(cache_key)
if not cached:
cached = self.get_related_assets().count()
cache.set(cache_key, cached, self.ASSET_USER_CACHE_TIME)
return cached
def expire_assets_amount(self):
cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id)
cache.delete(cache_key)
def get_asset_connectivity(self, asset):
i = self.generate_id_with_asset(asset)
@ -133,12 +160,15 @@ class AssetUser(OrgModelMixin):
i = self.generate_id_with_asset(asset)
key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
Connectivity.set(key, c)
# 当为某个系统用户或管理用户设置的的时候，失效掉他们的连接数量
amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
cache.delete(amount_key)
def get_asset_user(self, asset):
from ..backends import AssetUserManager
try:
manager = AssetUserManager().prefer(self._prefer)
other = manager.get(username=self.username, asset=asset)
other = manager.get(username=self.username, asset=asset, prefer_id=self.id)
return other
except Exception as e:
logger.error(e, exc_info=True)
@ -150,9 +180,12 @@ class AssetUser(OrgModelMixin):
self._merge_auth(instance)
def _merge_auth(self, other):
self.password = other.password
self.public_key = other.public_key
self.private_key = other.private_key
if other.password:
self.password = other.password
if other.public_key:
self.public_key = other.public_key
if other.private_key:
self.private_key = other.private_key
def clear_auth(self):
self.password = ''
@ -185,7 +218,7 @@ class AssetUser(OrgModelMixin):
}
def generate_id_with_asset(self, asset):
user_id = str(self.id).split('-')[:3]
user_id = [self.part_id]
asset_id = str(asset.id).split('-')[3:]
ids = user_id + asset_id
return '-'.join(ids)

3
apps/assets/models/node.py
View File

@ -192,6 +192,7 @@ class AssetsAmountMixin:
_assets_amount_cache_key = '_NODE_ASSETS_AMOUNT_{}'
_assets_amount = None
key = ''
cache_time = 3600 * 24 * 7
@property
def assets_amount(self):
@ -213,7 +214,7 @@ class AssetsAmountMixin:
def assets_amount(self, value):
self._assets_amount = value
cache_key = self._assets_amount_cache_key.format(self.key)
cache.set(cache_key, value)
cache.set(cache_key, value, self.cache_time)
def expire_assets_amount(self):
ancestor_keys = self.get_ancestor_keys(with_self=True)

2
apps/assets/serializers/admin_user.py
View File

@ -17,7 +17,7 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer):
"""
class Meta:
# list_serializer_class = AdaptedBulkListSerializer
list_serializer_class = AdaptedBulkListSerializer
model = AdminUser
fields = [
'id', 'name', 'username', 'password', 'private_key', 'public_key',

4
apps/assets/serializers/asset.py
View File

@ -76,8 +76,8 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('labels', 'nodes')\
.select_related('admin_user')
queryset = queryset.prefetch_related('labels', 'nodes', 'protocols')\
.select_related('admin_user', 'domain')
return queryset
@staticmethod

10
apps/assets/serializers/system_user.py
View File

@ -20,7 +20,7 @@ class SystemUserSerializer(BulkOrgResourceModelSerializer):
'id', 'name', 'username', 'password', 'public_key', 'private_key',
'login_mode', 'login_mode_display', 'priority', 'protocol',
'auto_push', 'cmd_filters', 'sudo', 'shell', 'comment', 'nodes',
'assets', 'assets_amount', 'connectivity_amount'
'assets_amount', 'connectivity_amount'
]
extra_kwargs = {
'password': {"write_only": True},
@ -32,6 +32,12 @@ class SystemUserSerializer(BulkOrgResourceModelSerializer):
'created_by': {'read_only': True},
}
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('cmd_filters', 'nodes')
return queryset
class SystemUserAuthSerializer(AuthSerializer):
"""
@ -47,8 +53,6 @@ class SystemUserAuthSerializer(AuthSerializer):
class SystemUserSimpleSerializer(serializers.ModelSerializer):
"""
系统用户最基本信息的数据结构

5
apps/assets/signals_handler.py
View File

@ -27,11 +27,6 @@ def test_asset_conn_on_created(asset):
test_asset_connectivity_util.delay([asset])
def set_asset_root_node(asset):
logger.debug("Set asset default node: {}".format(Node.root()))
asset.nodes.add(Node.root())
@receiver(post_save, sender=Asset, dispatch_uid="my_unique_identifier")
@on_transaction_commit
def on_asset_created_or_update(sender, instance=None, created=False, **kwargs):

2
apps/assets/templates/assets/admin_user_list.html
View File

@ -84,7 +84,7 @@ function initTable() {
$(td).html(innerHtml)
}},
{targets: 5, createdCell: function (td, cellData) {
var data = cellData['unreachable'];
var data = cellData.unreachable;
var innerHtml = "";
if (data !== 0) {
innerHtml = "<span class='text-danger'>" + data + "</span>";

9
apps/assets/templates/assets/system_user_list.html
View File

@ -80,7 +80,7 @@ function initTable() {
}},
{targets: 6, createdCell: function (td, cellData) {
var innerHtml = "";
var data = cellData['reachable'];
var data = cellData.reachable;
if (data !== 0) {
innerHtml = "<span class='text-navy'>" + data + "</span>";
} else {
@ -89,7 +89,7 @@ function initTable() {
$(td).html(innerHtml)
}},
{targets: 7, createdCell: function (td, cellData) {
var data = cellData['unreachable'];
var data = cellData.unreachable;
var innerHtml = "";
if (data !== 0) {
innerHtml = "<span class='text-danger'>" + data + "</span>";
@ -103,7 +103,7 @@ function initTable() {
var innerHtml = "";
var total = rowData.assets_amount;
var reachable = cellData.reachable;
if (total !== 0) {
if (total && total !== 0) {
val = reachable/total * 100;
}
@ -119,7 +119,8 @@ function initTable() {
var update_btn = '<a href="{% url "assets:system-user-update" pk=DEFAULT_PK %}" class="btn btn-xs m-l-xs btn-info">{% trans "Update" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn_admin_user_delete" data-uid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
$(td).html(update_btn + del_btn)
}}],
}},
],
ajax_url: '{% url "api-assets:system-user-list" %}',
columns: [
{data: "id" }, {data: "name" }, {data: "username" }, {data: "protocol"}, {data: "login_mode_display"}, {data: "assets_amount" },

5
apps/assets/views/asset.py
View File

@ -220,6 +220,11 @@ class AssetDetailView(PermissionsMixin, DetailView):
template_name = 'assets/asset_detail.html'
permission_classes = [IsValidUser]
def get_queryset(self):
return super().get_queryset().prefetch_related(
"nodes", "labels", "protocols"
).select_related('admin_user', 'domain')
def get_context_data(self, **kwargs):
nodes_remain = Node.objects.exclude(assets=self.object)
context = {

5
apps/assets/views/system_user.py
View File

@ -97,7 +97,10 @@ class SystemUserAssetView(PermissionsMixin, DetailView):
permission_classes = [IsOrgAdmin]
def get_context_data(self, **kwargs):
nodes_remain = sorted(Node.objects.exclude(systemuser=self.object), reverse=True)
from ..utils import NodeUtil
nodes_remain = Node.objects.exclude(systemuser=self.object)
util = NodeUtil()
nodes_remain = util.get_nodes_by_queryset(nodes_remain)
context = {
'app': _('assets'),
'action': _('System user asset'),

5
apps/common/signals_handlers.py
View File

@ -8,11 +8,12 @@ from django.core.signals import request_finished
from django.db import connection
from .utils import get_logger
from common.utils import get_logger
from .local import thread_local
logger = get_logger(__file__)
pattern = re.compile(r'FROM `(\w+)`')
# logger = logging.getLogger('jmsdb')
logger = get_logger(__name__)
class Counter:

6
apps/orgs/mixins/api.py
View File

@ -27,7 +27,11 @@ class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet):
class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet):
def get_queryset(self):
return super().get_queryset().all()
queryset = super().get_queryset().all()
if hasattr(self, 'serializer_class') and \
hasattr(self.serializer_class, 'setup_eager_loading'):
queryset = self.serializer_class.setup_eager_loading(queryset)
return queryset
class OrgMembershipModelViewSetMixin:

3
apps/terminal/api/session.py
View File

@ -31,7 +31,8 @@ class SessionViewSet(OrgBulkModelViewSet):
pagination_class = LimitOffsetPagination
permission_classes = (IsOrgAdminOrAppUser | IsAuditor, )
filter_fields = [
"user", "asset", "system_user", "terminal", "is_finished",
"user", "asset", "system_user", "remote_addr",
"protocol", "terminal", "is_finished",
]
date_range_filter_fields = [
('date_start', ('date_from', 'date_to'))

20
apps/terminal/models.py
View File

@ -191,26 +191,22 @@ class Session(OrgModelMixin):
@property
def _date_start_first_has_replay_rdp_session(self):
if self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None:
if self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None:
instance = self.__class__.objects.filter(
protocol='rdp', has_replay=True).order_by('date_start').first()
protocol='rdp', has_replay=True
).order_by('date_start').first()
if not instance:
return None
self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = instance.date_start
return self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION
date_start = timezone.now() - timezone.timedelta(days=365)
else:
date_start = instance.date_start
self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = date_start
return self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION
def can_replay(self):
if self.has_replay:
return True
# 判断对RDP Session添加上报has_replay状态机制之前的录像回放
if self._date_start_first_has_replay_rdp_session is None:
return True
if self.date_start < self._date_start_first_has_replay_rdp_session:
return True
return False
def save_to_storage(self, f):

13
apps/terminal/templates/terminal/command_list.html
View File

@ -71,7 +71,7 @@
{% block content_bottom_left %}{% endblock %}
{% block custom_foot_js %}
<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}" charset="UTF-8"></script>
<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.zh-CN.min.js' %}" charset="UTF-8"></script>
<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.zh-CN.min.js' %}" ></script>
<script>
var table;
@ -136,8 +136,8 @@ $(document).ready(function () {
$('.dropdown-menu.search-help').hide();
keyword.focus()
})
.on("click", "document", function () {
$('.dropdown-menu.search-help').hide();
.on('click', 'body', function (e) {
$('.dropdown-menu.search-help').hide()
})
.on('click', '.toggle', function (e) {
e.preventDefault();
@ -162,12 +162,7 @@ $(document).ready(function () {
detailRows.push(tr.attr('id'));
}
}
}).on('click', 'body', function (e) {
$('.dropdown-menu.search-help').hide()
}).on('change', '#date_start', function () {
console.log("date change")
});
})
function format(d) {

79
apps/terminal/templates/terminal/session_list.html
View File

@ -71,13 +71,15 @@
<li><a class="search-item" data-value="user">{% trans 'User' %}</a></li>
<li><a class="search-item" data-value="asset">{% trans 'Asset' %}</a></li>
<li><a class="search-item" data-value="system_user">{% trans 'System user' %}</a></li>
<li><a class="search-item" data-value="input">{% trans 'Command' %}</a></li>
<li><a class="search-item" data-value="remote_addr">{% trans 'Remote addr' %}</a></li>
<li><a class="search-item" data-value="protocol">{% trans 'Protocol' %}</a></li>
</ul>
{% endblock %}
{% block custom_foot_js %}
<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.zh-CN.min.js' %}" ></script>
<script>
function terminateSession(data) {
@ -182,19 +184,42 @@ function finishedSession(data) {
}
var table;
$(document).ready(function() {
table = initTable("#session_table");
$('.select2').select2({
dropdownAutoWidth: true,
width: "auto"
});
$('.input-daterange.input-group').datepicker({
format: "yyyy-mm-dd",
todayBtn: "linked",
keyboardNavigation: false,
forceParse: false,
calendarWeeks: true,
autoclose: true
table = initTable("#session_table").on('init', function () {
var dateFromRef = $("#date_from");
var dateToRef = $("#date_to");
var options = {
format: "yyyy-mm-dd",
todayBtn: "linked",
keyboardNavigation: false,
forceParse: false,
calendarWeeks: true,
autoclose: true,
language: navigator.language || "en",
};
dateFromRef.datepicker(options).on("changeDate", function () {
if (!$(this).val()) {
return
}
var value = $(this).val() + ' 0:0:0';
var date = new Date(value);
var url = table.ajax.url();
url = setUrlParam(url, "date_from", date.toISOString());
table.ajax.url(url);
table.ajax.reload();
});
dateToRef.datepicker(options).on("changeDate", function () {
if (!$(this).val()) {
return
}
var value = $(this).val() + ' 23:59:59';
var date = new Date(value);
var url = table.ajax.url();
url = setUrlParam(url, "date_to", date.toISOString());
table.ajax.url(url);
table.ajax.reload();
});
});
}).on('click', '.btn-term', function () {
var $this = $(this);
var session_id = $this.attr('value');
@ -207,6 +232,34 @@ $(document).ready(function() {
var replayUrl = "/luna/replay/" + sessionID;
window.open(replayUrl, "height=600, width=800, top=400, left=400, toolbar=no, menubar=no, scrollbars=no, location=no, status=no");
})
.on("click", '#session_table_filter input', function (e) {
e.preventDefault();
e.stopPropagation();
var offset1 = $('#session_table_filter input').offset();
var x = offset1.left;
var y = offset1.top;
var offset = $(".search-help").parent().offset();
x -= offset.left;
y -= offset.top;
x += 18;
y += 80;
$('.search-help').css({"top":y+"px", "left":x+"px", "position": "absolute"});
$('.dropdown-menu.search-help').show();
})
.on('click', '.search-item', function (e) {
e.preventDefault();
e.stopPropagation();
var keyword = $("#session_table_filter input");
var value = $(this).data('value');
var old_value = keyword.val();
var new_value = old_value + ' ' + value + ':';
keyword.val(new_value.trim());
$('.dropdown-menu.search-help').hide();
keyword.focus()
})
.on('click', 'body', function (e) {
$('.dropdown-menu.search-help').hide()
})
.on('click', '#btn_bulk_update', function () {
var action = $('#slct_bulk_update').val();
var idList = table.selected;