From a609f1707834aee935096bdb8802acc2cd857bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=81=E5=B9=BF?= Date: Tue, 18 Dec 2018 17:28:45 +0800 Subject: [PATCH] [Update] Stash it (#2197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] Stash it * [Bugfix] 修复错误 * [Update] 修改jms --- apps/assets/api/admin_user.py | 25 +- apps/assets/api/asset.py | 6 +- apps/assets/api/node.py | 4 +- apps/assets/api/system_user.py | 15 +- apps/assets/forms/user.py | 6 +- apps/assets/models/asset.py | 107 +++---- apps/assets/models/base.py | 7 + apps/assets/models/user.py | 99 ++++-- apps/assets/serializers/asset.py | 10 +- apps/assets/serializers/system_user.py | 15 +- apps/assets/signals_handler.py | 6 +- apps/assets/tasks.py | 289 +++++++----------- .../templates/assets/admin_user_assets.html | 47 ++- .../templates/assets/system_user_asset.html | 10 +- .../templates/assets/system_user_list.html | 2 +- apps/assets/urls/api_urls.py | 14 +- apps/assets/views/admin_user.py | 2 +- apps/jumpserver/conf.py | 3 +- apps/locale/zh/LC_MESSAGES/django.mo | Bin 60196 -> 60280 bytes apps/locale/zh/LC_MESSAGES/django.po | 18 +- apps/ops/api/adhoc.py | 4 +- apps/ops/views/adhoc.py | 2 +- apps/users/api/user.py | 3 + 23 files changed, 361 insertions(+), 333 deletions(-) diff --git a/apps/assets/api/admin_user.py b/apps/assets/api/admin_user.py index 8d30ee9d9..263d669fd 100644 --- a/apps/assets/api/admin_user.py +++ b/apps/assets/api/admin_user.py @@ -14,6 +14,7 @@ # limitations under the License. from django.db import transaction +from django.shortcuts import get_object_or_404 from rest_framework import generics from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet @@ -24,13 +25,14 @@ from common.utils import get_logger from ..hands import IsOrgAdmin from ..models import AdminUser, Asset from .. import serializers -from ..tasks import test_admin_user_connectability_manual +from ..tasks import test_admin_user_connectivity_manual logger = get_logger(__file__) __all__ = [ 'AdminUserViewSet', 'ReplaceNodesAdminUserApi', 'AdminUserTestConnectiveApi', 'AdminUserAuthApi', + 'AdminUserAssetsListView', ] @@ -81,12 +83,29 @@ class ReplaceNodesAdminUserApi(generics.UpdateAPIView): class AdminUserTestConnectiveApi(generics.RetrieveAPIView): """ - Test asset admin user connectivity + Test asset admin user assets_connectivity """ queryset = AdminUser.objects.all() permission_classes = (IsOrgAdmin,) def retrieve(self, request, *args, **kwargs): admin_user = self.get_object() - task = test_admin_user_connectability_manual.delay(admin_user) + task = test_admin_user_connectivity_manual.delay(admin_user) return Response({"task": task.id}) + + +class AdminUserAssetsListView(generics.ListAPIView): + permission_classes = (IsOrgAdmin,) + serializer_class = serializers.AssetSimpleSerializer + pagination_class = LimitOffsetPagination + filter_fields = ("hostname", "ip") + http_method_names = ['get'] + search_fields = filter_fields + + def get_object(self): + pk = self.kwargs.get('pk') + return get_object_or_404(AdminUser, pk=pk) + + def get_queryset(self): + admin_user = self.get_object() + return admin_user.get_related_assets() diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 986829def..cd343b2a5 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -17,7 +17,7 @@ from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from ..models import Asset, AdminUser, Node from .. import serializers from ..tasks import update_asset_hardware_info_manual, \ - test_asset_connectability_manual + test_asset_connectivity_manual from ..utils import LabelFilter @@ -109,7 +109,7 @@ class AssetRefreshHardwareApi(generics.RetrieveAPIView): class AssetAdminUserTestApi(generics.RetrieveAPIView): """ - Test asset admin user connectivity + Test asset admin user assets_connectivity """ queryset = Asset.objects.all() permission_classes = (IsOrgAdmin,) @@ -117,7 +117,7 @@ class AssetAdminUserTestApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): asset_id = kwargs.get('pk') asset = get_object_or_404(Asset, pk=asset_id) - task = test_asset_connectability_manual.delay(asset) + task = test_asset_connectivity_manual.delay(asset) return Response({"task": task.id}) diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 84ba4c69f..4295b2618 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -24,7 +24,7 @@ from common.utils import get_logger, get_object_or_none from common.tree import TreeNodeSerializer from ..hands import IsOrgAdmin from ..models import Node -from ..tasks import update_assets_hardware_info_util, test_asset_connectability_util +from ..tasks import update_assets_hardware_info_util, test_asset_connectivity_util from .. import serializers @@ -273,5 +273,5 @@ class TestNodeConnectiveApi(APIView): assets = node.assets.all() # task_name = _("测试节点下资产是否可连接: {}".format(node.name)) task_name = _("Test if the assets under the node are connectable: {}".format(node.name)) - task = test_asset_connectability_util.delay(assets, task_name=task_name) + task = test_asset_connectivity_util.delay(assets, task_name=task_name) return Response({"task": task.id}) diff --git a/apps/assets/api/system_user.py b/apps/assets/api/system_user.py index 3c1d0b3bd..e66e4bfc9 100644 --- a/apps/assets/api/system_user.py +++ b/apps/assets/api/system_user.py @@ -24,8 +24,8 @@ from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from ..models import SystemUser, Asset from .. import serializers from ..tasks import push_system_user_to_assets_manual, \ - test_system_user_connectability_manual, push_system_user_a_asset_manual, \ - test_system_user_connectability_a_asset + test_system_user_connectivity_manual, push_system_user_a_asset_manual, \ + test_system_user_connectivity_a_asset logger = get_logger(__file__) @@ -33,7 +33,7 @@ __all__ = [ 'SystemUserViewSet', 'SystemUserAuthInfoApi', 'SystemUserPushApi', 'SystemUserTestConnectiveApi', 'SystemUserAssetsListView', 'SystemUserPushToAssetApi', - 'SystemUserTestAssetConnectabilityApi', 'SystemUserCommandFilterRuleListApi', + 'SystemUserTestAssetConnectivityApi', 'SystemUserCommandFilterRuleListApi', ] @@ -93,15 +93,16 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): system_user = self.get_object() - task = test_system_user_connectability_manual.delay(system_user) + task = test_system_user_connectivity_manual.delay(system_user) return Response({"task": task.id}) class SystemUserAssetsListView(generics.ListAPIView): permission_classes = (IsOrgAdmin,) - serializer_class = serializers.AssetSerializer + serializer_class = serializers.AssetSimpleSerializer pagination_class = LimitOffsetPagination filter_fields = ("hostname", "ip") + http_method_names = ['get'] search_fields = filter_fields def get_object(self): @@ -125,7 +126,7 @@ class SystemUserPushToAssetApi(generics.RetrieveAPIView): return Response({"task": task.id}) -class SystemUserTestAssetConnectabilityApi(generics.RetrieveAPIView): +class SystemUserTestAssetConnectivityApi(generics.RetrieveAPIView): queryset = SystemUser.objects.all() permission_classes = (IsOrgAdmin,) @@ -133,7 +134,7 @@ class SystemUserTestAssetConnectabilityApi(generics.RetrieveAPIView): system_user = self.get_object() asset_id = self.kwargs.get('aid') asset = get_object_or_404(Asset, id=asset_id) - task = test_system_user_connectability_a_asset.delay(system_user, asset) + task = test_system_user_connectivity_a_asset.delay(system_user, asset) return Response({"task": task.id}) diff --git a/apps/assets/forms/user.py b/apps/assets/forms/user.py index f5c62a4ff..70fcdafad 100644 --- a/apps/assets/forms/user.py +++ b/apps/assets/forms/user.py @@ -99,8 +99,8 @@ class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): auto_generate_key = self.cleaned_data.get('auto_generate_key', False) private_key, public_key = super().gen_keys() - if login_mode == SystemUser.MANUAL_LOGIN or \ - protocol in [SystemUser.RDP_PROTOCOL, SystemUser.TELNET_PROTOCOL]: + if login_mode == SystemUser.LOGIN_MANUAL or \ + protocol in [SystemUser.PROTOCOL_RDP, SystemUser.PROTOCOL_TELNET]: system_user.auto_push = 0 auto_generate_key = False system_user.save() @@ -124,7 +124,7 @@ class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): validated = super().is_valid() username = self.cleaned_data.get('username') login_mode = self.cleaned_data.get('login_mode') - if login_mode == SystemUser.AUTO_LOGIN and not username: + if login_mode == SystemUser.LOGIN_AUTO and not username: self.add_error( "username", _('* Automatic login mode,' ' must fill in the username.') diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 06fb29a51..ccff0fff5 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -13,7 +13,6 @@ from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.core.cache import cache -from ..const import ASSET_ADMIN_CONN_CACHE_KEY from .user import AdminUser, SystemUser from orgs.mixins import OrgModelMixin, OrgManager @@ -75,63 +74,48 @@ class Asset(OrgModelMixin): protocol = models.CharField(max_length=128, default=SSH_PROTOCOL, choices=PROTOCOL_CHOICES, verbose_name=_('Protocol')) port = models.IntegerField(default=22, verbose_name=_('Port')) platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES, default='Linux', verbose_name=_('Platform')) - domain = models.ForeignKey("assets.Domain", null=True, blank=True, - related_name='assets', verbose_name=_("Domain"), - on_delete=models.SET_NULL) - nodes = models.ManyToManyField('assets.Node', default=default_node, - related_name='assets', - verbose_name=_("Nodes")) + domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets', verbose_name=_("Domain"), on_delete=models.SET_NULL) + nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes")) is_active = models.BooleanField(default=True, verbose_name=_('Is active')) # Auth - admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, - null=True, verbose_name=_("Admin user")) + admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, null=True, verbose_name=_("Admin user")) # Some information public_ip = models.GenericIPAddressField(max_length=32, blank=True, null=True, verbose_name=_('Public IP')) number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number')) # Collect - vendor = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('Vendor')) - model = models.CharField(max_length=54, null=True, blank=True, - verbose_name=_('Model')) - sn = models.CharField(max_length=128, null=True, blank=True, - verbose_name=_('Serial number')) + vendor = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Vendor')) + model = models.CharField(max_length=54, null=True, blank=True, verbose_name=_('Model')) + sn = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Serial number')) - cpu_model = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('CPU model')) + cpu_model = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU model')) cpu_count = models.IntegerField(null=True, verbose_name=_('CPU count')) cpu_cores = models.IntegerField(null=True, verbose_name=_('CPU cores')) cpu_vcpus = models.IntegerField(null=True, verbose_name=_('CPU vcpus')) - memory = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('Memory')) - disk_total = models.CharField(max_length=1024, null=True, blank=True, - verbose_name=_('Disk total')) - disk_info = models.CharField(max_length=1024, null=True, blank=True, - verbose_name=_('Disk info')) + memory = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Memory')) + disk_total = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk total')) + disk_info = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk info')) - os = models.CharField(max_length=128, null=True, blank=True, - verbose_name=_('OS')) - os_version = models.CharField(max_length=16, null=True, blank=True, - verbose_name=_('OS version')) - os_arch = models.CharField(max_length=16, blank=True, null=True, - verbose_name=_('OS arch')) - hostname_raw = models.CharField(max_length=128, blank=True, null=True, - verbose_name=_('Hostname raw')) + os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS')) + os_version = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('OS version')) + os_arch = models.CharField(max_length=16, blank=True, null=True, verbose_name=_('OS arch')) + hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw')) - labels = models.ManyToManyField('assets.Label', blank=True, - related_name='assets', - verbose_name=_("Labels")) - created_by = models.CharField(max_length=32, null=True, blank=True, - verbose_name=_('Created by')) - date_created = models.DateTimeField(auto_now_add=True, null=True, - blank=True, - verbose_name=_('Date created')) - comment = models.TextField(max_length=128, default='', blank=True, - verbose_name=_('Comment')) + labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels")) + created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by')) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created')) + comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment')) objects = OrgManager.from_queryset(AssetQuerySet)() + CONNECTIVITY_CACHE_KEY = '_JMS_ASSET_CONNECTIVITY_{}' + UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) + CONNECTIVITY_CHOICES = ( + (UNREACHABLE, _("Unreachable")), + (REACHABLE, _('Reachable')), + (UNKNOWN, _("Unknown")), + ) def __str__(self): return '{0.hostname}({0.ip})'.format(self) @@ -197,25 +181,17 @@ class Asset(OrgModelMixin): return '' @property - def is_connective(self): + def connectivity(self): if not self.is_unixlike(): - return True - val = cache.get(ASSET_ADMIN_CONN_CACHE_KEY.format(self.hostname)) - if val == 1: - return True - else: - return False + return self.UNKNOWN + key = self.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + cached = cache.get(key, None) + return cached if cached is not None else self.UNKNOWN - def to_json(self): - info = { - 'id': self.id, - 'hostname': self.hostname, - 'ip': self.ip, - 'port': self.port, - } - if self.domain and self.domain.gateway_set.all(): - info["gateways"] = [d.id for d in self.domain.gateway_set.all()] - return info + @connectivity.setter + def connectivity(self, value): + key = self.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + cache.set(key, value, 3600*2) def get_auth_info(self): if self.admin_user: @@ -236,11 +212,20 @@ class Asset(OrgModelMixin): fake_node.is_node = False return fake_node + def to_json(self): + info = { + 'id': self.id, + 'hostname': self.hostname, + 'ip': self.ip, + 'port': self.port, + } + if self.domain and self.domain.gateway_set.all(): + info["gateways"] = [d.id for d in self.domain.gateway_set.all()] + return info + def _to_secret_json(self): """ - Ansible use it create inventory, First using asset user, - otherwise using cluster admin user - + Ansible use it create inventory Todo: May be move to ops implements it """ data = self.to_json() diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index af59f000c..37e099e99 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -29,6 +29,13 @@ class AssetUser(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) + UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) + CONNECTIVITY_CHOICES = ( + (UNREACHABLE, _("Unreachable")), + (REACHABLE, _('Reachable')), + (UNKNOWN, _("Unknown")), + ) + @property def password(self): if self._password: diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index f5c8e17a1..147687471 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -14,7 +14,7 @@ from ..const import SYSTEM_USER_CONN_CACHE_KEY from .base import AssetUser -__all__ = ['AdminUser', 'SystemUser',] +__all__ = ['AdminUser', 'SystemUser'] logger = logging.getLogger(__name__) signer = get_signer() @@ -31,6 +31,7 @@ class AdminUser(AssetUser): become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4) become_user = models.CharField(default='root', max_length=64) _become_pass = models.CharField(default='', max_length=128) + CONNECTIVE_CACHE_KEY = '_JMS_ADMIN_USER_CONNECTIVE_{}' def __str__(self): return self.name @@ -67,6 +68,23 @@ class AdminUser(AssetUser): def assets_amount(self): return self.get_related_assets().count() + @property + def connectivity(self): + from .asset import Asset + assets = self.get_related_assets().values_list('id', 'hostname', flat=True) + data = { + 'unreachable': [], + 'reachable': [], + } + for asset_id, hostname in assets: + key = Asset.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + value = cache.get(key, Asset.UNKNOWN) + if value == Asset.REACHABLE: + data['reachable'].append(hostname) + elif value == Asset.UNREACHABLE: + data['unreachable'].append(hostname) + return data + class Meta: ordering = ['name'] unique_together = [('name', 'org_id')] @@ -94,34 +112,34 @@ class AdminUser(AssetUser): class SystemUser(AssetUser): - SSH_PROTOCOL = 'ssh' - RDP_PROTOCOL = 'rdp' - TELNET_PROTOCOL = 'telnet' + PROTOCOL_SSH = 'ssh' + PROTOCOL_RDP = 'rdp' + PROTOCOL_TELNET = 'telnet' PROTOCOL_CHOICES = ( - (SSH_PROTOCOL, 'ssh'), - (RDP_PROTOCOL, 'rdp'), - (TELNET_PROTOCOL, 'telnet (beta)'), + (PROTOCOL_SSH, 'ssh'), + (PROTOCOL_RDP, 'rdp'), + (PROTOCOL_TELNET, 'telnet (beta)'), ) - AUTO_LOGIN = 'auto' - MANUAL_LOGIN = 'manual' + LOGIN_AUTO = 'auto' + LOGIN_MANUAL = 'manual' LOGIN_MODE_CHOICES = ( - (AUTO_LOGIN, _('Automatic login')), - (MANUAL_LOGIN, _('Manually login')) + (LOGIN_AUTO, _('Automatic login')), + (LOGIN_MANUAL, _('Manually login')) ) nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes")) assets = models.ManyToManyField('assets.Asset', blank=True, verbose_name=_("Assets")) - priority = models.IntegerField(default=20, verbose_name=_("Priority"), - validators=[MinValueValidator(1), MaxValueValidator(100)]) + priority = models.IntegerField(default=20, verbose_name=_("Priority"), validators=[MinValueValidator(1), MaxValueValidator(100)]) protocol = models.CharField(max_length=16, choices=PROTOCOL_CHOICES, default='ssh', verbose_name=_('Protocol')) auto_push = models.BooleanField(default=True, verbose_name=_('Auto push')) sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo')) shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell')) - login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=AUTO_LOGIN, max_length=10, verbose_name=_('Login mode')) + login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')) cmd_filters = models.ManyToManyField('CommandFilter', related_name='system_users', verbose_name=_("Command filter"), blank=True) - cache_key = "__SYSTEM_USER_CACHED_{}" + SYSTEM_USER_CACHE_KEY = "__SYSTEM_USER_CACHED_{}" + CONNECTIVE_CACHE_KEY = '_JMS_SYSTEM_USER_CONNECTIVE_{}' def __str__(self): return '{0.name}({0.username})'.format(self) @@ -136,34 +154,61 @@ class SystemUser(AssetUser): 'auto_push': self.auto_push, } - def get_assets(self): + def get_related_assets(self): assets = set(self.assets.all()) return assets @property - def assets_connective(self): - _result = cache.get(SYSTEM_USER_CONN_CACHE_KEY.format(self.name), {}) - return _result + def connectivity(self): + cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) + value = cache.get(cache_key, None) + if not value or 'unreachable' not in value: + return {'unreachable': [], 'reachable': []} + else: + return value + + @connectivity.setter + def connectivity(self, value): + data = self.connectivity + unreachable = data['unreachable'] + reachable = data['reachable'] + + for host in value.get('dark', {}).keys(): + if host not in unreachable: + unreachable.append(host) + if host in reachable: + reachable.remove(host) + for host in value.get('contacted'): + if host not in reachable: + reachable.append(host) + if host in unreachable: + unreachable.remove(host) + cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) + cache.set(cache_key, data, 3600) @property - def unreachable_assets(self): - return list(self.assets_connective.get('dark', {}).keys()) + def assets_unreachable(self): + return self.connectivity.get('unreachable') @property - def reachable_assets(self): - return self.assets_connective.get('contacted', []) + def assets_reachable(self): + return self.connectivity.get('reachable') + + @property + def login_mode_display(self): + return self.get_login_mode_display() def is_need_push(self): - if self.auto_push and self.protocol == self.__class__.SSH_PROTOCOL: + if self.auto_push and self.protocol == self.PROTOCOL_SSH: return True else: return False def set_cache(self): - cache.set(self.cache_key.format(self.id), self, 3600) + cache.set(self.SYSTEM_USER_CACHE_KEY.format(self.id), self, 3600) def expire_cache(self): - cache.delete(self.cache_key.format(self.id)) + cache.delete(self.SYSTEM_USER_CACHE_KEY.format(self.id)) @property def cmd_filter_rules(self): @@ -184,7 +229,7 @@ class SystemUser(AssetUser): @classmethod def get_system_user_by_id_or_cached(cls, sid): - cached = cache.get(cls.cache_key.format(sid)) + cached = cache.get(cls.SYSTEM_USER_CACHE_KEY.format(sid)) if cached: return cached try: diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index 1066ae0b7..9640aff7f 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -9,7 +9,7 @@ from .system_user import AssetSystemUserSerializer __all__ = [ 'AssetSerializer', 'AssetGrantedSerializer', 'MyAssetGrantedSerializer', - 'AssetAsNodeSerializer', + 'AssetAsNodeSerializer', 'AssetSimpleSerializer', ] @@ -33,7 +33,7 @@ class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer): def get_field_names(self, declared_fields, info): fields = super().get_field_names(declared_fields, info) fields.extend([ - 'hardware_info', 'is_connective', 'org_name' + 'hardware_info', 'connectivity', 'org_name' ]) return fields @@ -78,3 +78,9 @@ class MyAssetGrantedSerializer(AssetGrantedSerializer): "is_active", "system_users_join", "org_name", "os", "platform", "comment", "org_id", "protocol" ) + + +class AssetSimpleSerializer(serializers.ModelSerializer): + class Meta: + model = Asset + fields = ['id', 'hostname', 'port', 'ip', 'connectivity'] diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index a295f245c..be1f594ec 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from ..models import SystemUser +from ..models import SystemUser, Asset from .base import AuthSerializer @@ -21,17 +21,17 @@ class SystemUserSerializer(serializers.ModelSerializer): def get_field_names(self, declared_fields, info): fields = super(SystemUserSerializer, self).get_field_names(declared_fields, info) fields.extend([ - 'get_login_mode_display', + 'login_mode_display', ]) return fields @staticmethod def get_unreachable_assets(obj): - return obj.unreachable_assets + return obj.assets_unreachable @staticmethod def get_reachable_assets(obj): - return obj.reachable_assets + return obj.assets_reachable def get_unreachable_amount(self, obj): return len(self.get_unreachable_assets(obj)) @@ -41,7 +41,7 @@ class SystemUserSerializer(serializers.ModelSerializer): @staticmethod def get_assets_amount(obj): - return len(obj.get_assets()) + return len(obj.get_related_assets()) class SystemUserAuthSerializer(AuthSerializer): @@ -75,4 +75,7 @@ class SystemUserSimpleSerializer(serializers.ModelSerializer): """ class Meta: model = SystemUser - fields = ('id', 'name', 'username') \ No newline at end of file + fields = ('id', 'name', 'username') + + + diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 08ee6e670..85156c60d 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -7,7 +7,7 @@ from django.dispatch import receiver from common.utils import get_logger from .models import Asset, SystemUser, Node from .tasks import update_assets_hardware_info_util, \ - test_asset_connectability_util, push_system_user_to_assets + test_asset_connectivity_util, push_system_user_to_assets logger = get_logger(__file__) @@ -19,8 +19,8 @@ def update_asset_hardware_info_on_created(asset): def test_asset_conn_on_created(asset): - logger.debug("Test asset `{}` connectability".format(asset)) - test_asset_connectability_util.delay([asset]) + logger.debug("Test asset `{}` connectivity".format(asset)) + test_asset_connectivity_util.delay([asset]) def set_asset_root_node(asset): diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py index 846964d94..0e6a6ec99 100644 --- a/apps/assets/tasks.py +++ b/apps/assets/tasks.py @@ -26,6 +26,23 @@ disk_pattern = re.compile(r'^hd|sd|xvd|vd') PERIOD_TASK = os.environ.get("PERIOD_TASK", "off") +def clean_hosts(assets): + clean_assets = [] + for asset in assets: + if not asset.is_active: + msg = _("Asset has been disabled, skipped: {}").format(asset) + logger.info(msg) + continue + if not asset.support_ansible(): + msg = _("Asset may not be support ansible, skipped: {}").format(asset) + logger.info(msg) + continue + clean_assets.append(asset) + if not clean_assets: + logger.info(_("No assets matched, stop task")) + return clean_assets + + @shared_task def set_assets_hardware_info(assets, result, **kwargs): """ @@ -60,9 +77,12 @@ def set_assets_hardware_info(assets, result, **kwargs): ___cpu_model = 'Unknown' ___cpu_model = ___cpu_model[:64] ___cpu_count = info.get('ansible_processor_count', 0) - ___cpu_cores = info.get('ansible_processor_cores', None) or len(info.get('ansible_processor', [])) + ___cpu_cores = info.get('ansible_processor_cores', None) or \ + len(info.get('ansible_processor', [])) ___cpu_vcpus = info.get('ansible_processor_vcpus', 0) - ___memory = '%s %s' % capacity_convert('{} MB'.format(info.get('ansible_memtotal_mb'))) + ___memory = '%s %s' % capacity_convert( + '{} MB'.format(info.get('ansible_memtotal_mb')) + ) disk_info = {} for dev, dev_info in info.get('ansible_devices', {}).items(): if disk_pattern.match(dev) and dev_info['removable'] == '0': @@ -96,19 +116,8 @@ def update_assets_hardware_info_util(assets, task_name=None): if task_name is None: task_name = _("Update some assets hardware info") tasks = const.UPDATE_ASSETS_HARDWARE_TASKS - hosts = [] - for asset in assets: - if not asset.is_active: - msg = _("Asset has been disabled, skipped: {}").format(asset) - logger.info(msg) - continue - if not asset.support_ansible(): - msg = _("Asset may not be support ansible, skipped: {}").format(asset) - logger.info(msg) - continue - hosts.append(asset) + hosts = clean_hosts(assets) if not hosts: - logger.info(_("No assets matched, stop task")) return {} created_by = str(assets[0].org_id) task, created = update_or_create_ansible_task( @@ -125,7 +134,6 @@ def update_assets_hardware_info_util(assets, task_name=None): @shared_task def update_asset_hardware_info_manual(asset): task_name = _("Update asset hardware info: {}").format(asset.hostname) - # task_name = _("更新资产硬件信息") return update_assets_hardware_info_util( [asset], task_name=task_name ) @@ -141,123 +149,18 @@ def update_assets_hardware_info_period(): logger.debug("Period task disabled, update assets hardware info pass") return - # from ops.utils import update_or_create_ansible_task - # from orgs.models import Organization - # orgs = Organization.objects.all().values_list('id', flat=True) - # orgs.append('') - # task_name = _("Update assets hardware info period") - # for org_id in orgs: - # org_id = str(org_id) - # hostname_list = [ - # asset for asset in Asset.objects.all() - # if asset.is_active and asset.is_unixlike() - # ] - # tasks = const.UPDATE_ASSETS_HARDWARE_TASKS - # - # # Only create, schedule by celery beat - # update_or_create_ansible_task( - # task_name, hosts=hostname_list, tasks=tasks, pattern='all', - # options=const.TASK_OPTIONS, run_as_admin=True, created_by='System', - # interval=60*60*24, is_periodic=True, callback=set_assets_hardware_info.name, - # ) - ## ADMIN USER CONNECTIVE ## -def set_admin_user_connectability_info(result, **kwargs): - admin_user = kwargs.get("admin_user") - task_name = kwargs.get("task_name") - if admin_user is None and task_name is not None: - admin_user = task_name.split(":")[-1] - - raw, summary = result - cache_key = const.ADMIN_USER_CONN_CACHE_KEY.format(admin_user) - cache.set(cache_key, summary, CACHE_MAX_TIME) - - for i in summary.get('contacted', []): - asset_conn_cache_key = const.ASSET_ADMIN_CONN_CACHE_KEY.format(i) - cache.set(asset_conn_cache_key, 1, CACHE_MAX_TIME) - - for i, msg in summary.get('dark', {}).items(): - asset_conn_cache_key = const.ASSET_ADMIN_CONN_CACHE_KEY.format(i) - cache.set(asset_conn_cache_key, 0, CACHE_MAX_TIME) - logger.error(msg) - @shared_task -def test_admin_user_connectability_util(admin_user, task_name): - """ - Test asset admin user can connect or not. Using ansible api do that - :param admin_user: - :param task_name: - :return: - """ - from ops.utils import update_or_create_ansible_task - - assets = admin_user.get_related_assets() - hosts = [] - for asset in assets: - if not asset.is_active: - msg = _("Asset has been disabled, skipped: {}").format(asset) - logger.info(msg) - continue - if not asset.support_ansible(): - msg = _("Asset may not be support ansible, skipped: {}").format(asset) - logger.info(msg) - continue - hosts.append(asset) - if not hosts: - logger.info(_("No assets matched, stop task")) - return {} - tasks = const.TEST_ADMIN_USER_CONN_TASKS - task, created = update_or_create_ansible_task( - task_name=task_name, hosts=hosts, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by=admin_user.org_id, - ) - result = task.run() - set_admin_user_connectability_info(result, admin_user=admin_user.name) - return result - - -@shared_task -@register_as_period_task(interval=3600) -def test_admin_user_connectability_period(): - """ - A period task that update the ansible task period - """ - admin_users = AdminUser.objects.all() - for admin_user in admin_users: - task_name = _("Test admin user connectability period: {}").format(admin_user.name) - test_admin_user_connectability_util(admin_user, task_name) - - -@shared_task -def test_admin_user_connectability_manual(admin_user): - task_name = _("Test admin user connectability: {}").format(admin_user.name) - # task_name = _("测试管理行号可连接性: {}").format(admin_user.name) - return test_admin_user_connectability_util(admin_user, task_name) - - -@shared_task -def test_asset_connectability_util(assets, task_name=None): +def test_asset_connectivity_util(assets, task_name=None): from ops.utils import update_or_create_ansible_task if task_name is None: - task_name = _("Test assets connectability") - # task_name = _("测试资产可连接性") - hosts = [] - for asset in assets: - if not asset.is_active: - msg = _("Asset has been disabled, skip: {}").format(asset) - logger.info(msg) - continue - if not asset.support_ansible(): - msg = _("Asset may not be support ansible, skip: {}").format(asset) - logger.info(msg) - continue - hosts.append(asset) + task_name = _("Test assets connectivity") + hosts = clean_hosts(assets) if not hosts: - logger.info(_("No assets, task stop")) return {} tasks = const.TEST_ADMIN_USER_CONN_TASKS created_by = assets[0].org_id @@ -267,18 +170,20 @@ def test_asset_connectability_util(assets, task_name=None): ) result = task.run() summary = result[1] - for k in summary.get('dark'): - cache.set(const.ASSET_ADMIN_CONN_CACHE_KEY.format(k), 0, CACHE_MAX_TIME) - - for k in summary.get('contacted'): - cache.set(const.ASSET_ADMIN_CONN_CACHE_KEY.format(k), 1, CACHE_MAX_TIME) + for asset in assets: + if asset.hostname in summary.get('dark', {}): + asset.connectivity = asset.UNREACHABLE + elif asset.hostname in summary.get('contacted', []): + asset.connectivity = asset.REACHABLE + else: + asset.connectivity = asset.UNKNOWN return summary @shared_task -def test_asset_connectability_manual(asset): - task_name = _("Test assets connectability: {}").format(asset) - summary = test_asset_connectability_util([asset], task_name=task_name) +def test_asset_connectivity_manual(asset): + task_name = _("Test assets connectivity: {}").format(asset) + summary = test_asset_connectivity_util([asset], task_name=task_name) if summary.get('dark'): return False, summary['dark'] @@ -286,21 +191,50 @@ def test_asset_connectability_manual(asset): return True, "" +@shared_task +def test_admin_user_connectivity_util(admin_user, task_name): + """ + Test asset admin user can connect or not. Using ansible api do that + :param admin_user: + :param task_name: + :return: + """ + assets = admin_user.get_related_assets() + hosts = clean_hosts(assets) + if not hosts: + return {} + summary = test_asset_connectivity_util(hosts, task_name) + return summary + + +@shared_task +@register_as_period_task(interval=3600) +def test_admin_user_connectivity_period(): + """ + A period task that update the ansible task period + """ + admin_users = AdminUser.objects.all() + for admin_user in admin_users: + task_name = _("Test admin user connectivity period: {}").format(admin_user.name) + test_admin_user_connectivity_util(admin_user, task_name) + + +@shared_task +def test_admin_user_connectivity_manual(admin_user): + task_name = _("Test admin user connectivity: {}").format(admin_user.name) + return test_admin_user_connectivity_util(admin_user, task_name) + + ## System user connective ## @shared_task -def set_system_user_connectablity_info(result, **kwargs): +def set_system_user_connectivity_info(system_user, result): summary = result[1] - task_name = kwargs.get("task_name") - system_user = kwargs.get("system_user") - if system_user is None: - system_user = task_name.split(":")[-1] - cache_key = const.SYSTEM_USER_CONN_CACHE_KEY.format(str(system_user.id)) - cache.set(cache_key, summary, CACHE_MAX_TIME) + system_user.connectivity = summary @shared_task -def test_system_user_connectability_util(system_user, assets, task_name): +def test_system_user_connectivity_util(system_user, assets, task_name): """ Test system cant connect his assets or not. :param system_user: @@ -309,20 +243,9 @@ def test_system_user_connectability_util(system_user, assets, task_name): :return: """ from ops.utils import update_or_create_ansible_task - hosts = [] tasks = const.TEST_SYSTEM_USER_CONN_TASKS - for asset in assets: - if not asset.is_active: - msg = _("Asset has been disabled, skip: {}").format(asset) - logger.info(msg) - continue - if not asset.support_ansible(): - msg = _("Asset may not be support ansible, skip: {}").format(asset) - logger.info(msg) - continue - hosts.append(asset) + hosts = clean_hosts(assets) if not hosts: - logger.info(_("No assets matched, stop task")) return {} task, created = update_or_create_ansible_task( task_name, hosts=hosts, tasks=tasks, pattern='all', @@ -330,36 +253,35 @@ def test_system_user_connectability_util(system_user, assets, task_name): run_as=system_user, created_by=system_user.org_id, ) result = task.run() - set_system_user_connectablity_info(result, system_user=system_user) + set_system_user_connectivity_info(system_user, result) return result @shared_task -def test_system_user_connectability_manual(system_user): - task_name = _("Test system user connectability: {}").format(system_user) - assets = system_user.get_assets() - return test_system_user_connectability_util(system_user, assets, task_name) +def test_system_user_connectivity_manual(system_user): + task_name = _("Test system user connectivity: {}").format(system_user) + assets = system_user.get_related_assets() + return test_system_user_connectivity_util(system_user, assets, task_name) @shared_task -def test_system_user_connectability_a_asset(system_user, asset): - task_name = _("Test system user connectability: {} => {}").format( +def test_system_user_connectivity_a_asset(system_user, asset): + task_name = _("Test system user connectivity: {} => {}").format( system_user, asset ) - return test_system_user_connectability_util(system_user, [asset], task_name) + return test_system_user_connectivity_util(system_user, [asset], task_name) @shared_task -def test_system_user_connectability_period(): +def test_system_user_connectivity_period(): if PERIOD_TASK != "on": - logger.debug("Period task disabled, test system user connectability pass") + logger.debug("Period task disabled, test system user connectivity pass") return - # Todo: 暂时禁用定期测试 - # system_users = SystemUser.objects.all() - # for system_user in system_users: - # task_name = _("Test system user connectability period: {}").format(system_user) - # # task_name = _("定期测试系统用户可连接性: {}".format(system_user)) - # test_system_user_connectability_util(system_user, task_name) + system_users = SystemUser.objects.all() + for system_user in system_users: + task_name = _("Test system user connectivity period: {}").format(system_user) + assets = system_user.get_related_assets() + test_system_user_connectivity_util(system_user, assets, task_name) #### Push system user tasks #### @@ -381,6 +303,24 @@ def get_push_system_user_tasks(system_user): ), } }) + tasks.extend([ + { + 'name': 'Check home dir exists', + 'action': { + 'module': 'stat', + 'args': 'path=/home/{}'.format(system_user.username) + }, + 'register': 'home_existed' + }, + { + 'name': "Set home dir permission", + 'action': { + 'module': 'file', + 'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username) + }, + 'when': 'home_existed.stat.exists == true' + } + ]) if system_user.public_key: tasks.append({ 'name': 'Set {} authorized key'.format(system_user.username), @@ -417,19 +357,8 @@ def push_system_user_util(system_user, assets, task_name): return tasks = get_push_system_user_tasks(system_user) - hosts = [] - for asset in assets: - if not asset.is_active: - msg = _("Asset has been disabled, skip: {}").format(asset) - logger.info(msg) - continue - if not asset.support_ansible(): - msg = _("Asset may not be support ansible, skip: {}").format(asset) - logger.info(msg) - continue - hosts.append(asset) + hosts = clean_hosts(assets) if not hosts: - logger.info(_("No assets matched, stop task")) return {} task, created = update_or_create_ansible_task( task_name=task_name, hosts=hosts, tasks=tasks, pattern='all', @@ -441,7 +370,7 @@ def push_system_user_util(system_user, assets, task_name): @shared_task def push_system_user_to_assets_manual(system_user): - assets = system_user.get_assets() + assets = system_user.get_related_assets() task_name = _("Push system users to assets: {}").format(system_user.name) return push_system_user_util(system_user, assets, task_name=task_name) diff --git a/apps/assets/templates/assets/admin_user_assets.html b/apps/assets/templates/assets/admin_user_assets.html index 31314392f..7a9882563 100644 --- a/apps/assets/templates/assets/admin_user_assets.html +++ b/apps/assets/templates/assets/admin_user_assets.html @@ -45,13 +45,11 @@ - + @@ -91,26 +89,36 @@
- - {% trans 'Hostname' %} {% trans 'IP' %} {% trans 'Port' %} {% trans 'Reachable' %}{% trans 'Action' %}