diff --git a/apps/assets/const.py b/apps/assets/const.py index eebb5ecca..e5f65fce0 100644 --- a/apps/assets/const.py +++ b/apps/assets/const.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # - +from django.utils.translation import ugettext_lazy as _ UPDATE_ASSETS_HARDWARE_TASKS = [ { @@ -11,7 +11,6 @@ UPDATE_ASSETS_HARDWARE_TASKS = [ } ] -ADMIN_USER_CONN_CACHE_KEY = "ADMIN_USER_CONN_{}" TEST_ADMIN_USER_CONN_TASKS = [ { "name": "ping", @@ -49,7 +48,6 @@ TEST_WINDOWS_SYSTEM_USER_CONN_TASKS = [ } ] -ASSET_USER_CONN_CACHE_KEY = 'ASSET_USER_CONN_{}' TEST_ASSET_USER_CONN_TASKS = [ { "name": "ping", @@ -74,5 +72,10 @@ TASK_OPTIONS = { } CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX = '_KEY_ASSET_BULK_UPDATE_ID_{}' - +CONN_UNREACHABLE, CONN_REACHABLE, CONN_UNKNOWN = range(0, 3) +CONNECTIVITY_CHOICES = ( + (CONN_UNREACHABLE, _("Unreachable")), + (CONN_REACHABLE, _('Reachable')), + (CONN_UNKNOWN, _("Unknown")), +) diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 7042340fe..1e72d2614 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -6,16 +6,14 @@ import uuid import logging import random from functools import reduce -from collections import defaultdict from django.db import models -from django.db.models import Q from django.utils.translation import ugettext_lazy as _ -from django.core.cache import cache from django.core.validators import MinValueValidator, MaxValueValidator from .user import AdminUser, SystemUser from orgs.mixins import OrgModelMixin, OrgManager +from ..utils import Connectivity __all__ = ['Asset', 'Protocol'] logger = logging.getLogger(__name__) @@ -230,12 +228,12 @@ class Asset(OrgModelMixin): @property def connectivity(self): if not self.admin_user: - return self.UNKNOWN - return self.admin_user.get_connectivity_of(self) + return Connectivity.unknown() + return self.admin_user.get_asset_connectivity(self) @connectivity.setter def connectivity(self, value): - self.admin_user.set_connectivity_of(self, value) + self.admin_user.set_asset_connectivity(self, value) def get_auth_info(self): if not self.admin_user: diff --git a/apps/assets/models/authbook.py b/apps/assets/models/authbook.py index e61c3423d..ba676c309 100644 --- a/apps/assets/models/authbook.py +++ b/apps/assets/models/authbook.py @@ -3,12 +3,10 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -from django.core.cache import cache from orgs.mixins import OrgManager - +from ..utils import Connectivity from .base import AssetUser -from ..const import ASSET_USER_CONN_CACHE_KEY __all__ = ['AuthBook'] @@ -32,6 +30,7 @@ class AuthBook(AssetUser): backend = "db" # 用于system user和admin_user的动态设置 _connectivity = None + CONN_CACHE_KEY = "ASSET_USER_CONN_{}" class Meta: verbose_name = _('AuthBook') @@ -65,20 +64,17 @@ class AuthBook(AssetUser): self._set_version() self._set_latest() - @property - def _conn_cache_key(self): - return ASSET_USER_CONN_CACHE_KEY.format(self.id) + def get_related_assets(self): + return [self.asset] + + def generate_id_with_asset(self, asset): + return self.id @property def connectivity(self): if self._connectivity: return self._connectivity - value = cache.get(self._conn_cache_key, self.UNKNOWN) - return value - - @connectivity.setter - def connectivity(self, value): - cache.set(self._conn_cache_key, value, 3600) + return self.get_asset_connectivity(self.asset) @property def keyword(self): diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index d58b97a18..5b9aeddde 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -14,8 +14,11 @@ from common.utils import ( get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger ) from common.validators import alphanumeric +from common import fields from orgs.mixins import OrgModelMixin from .utils import private_key_validator +from ..utils import Connectivity +from .. import const signer = get_signer() @@ -26,7 +29,7 @@ class AssetUser(OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_('Name')) username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric]) - _password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password')) + _password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password')) _private_key = models.TextField(max_length=4096, blank=True, null=True, verbose_name=_('SSH private key'), validators=[private_key_validator, ]) _public_key = models.TextField(max_length=4096, blank=True, verbose_name=_('SSH public key')) comment = models.TextField(blank=True, verbose_name=_('Comment')) @@ -34,13 +37,8 @@ class AssetUser(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) - UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) - CONNECTIVITY_CHOICES = ( - (UNREACHABLE, _("Unreachable")), - (REACHABLE, _('Reachable')), - (UNKNOWN, _("Unknown")), - ) - CONNECTIVITY_CACHE_KEY = "CONNECTIVITY_{}" + CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_ASSET_CONNECTIVITY_{}" + _prefer = "system_user" @property @@ -109,6 +107,10 @@ class AssetUser(OrgModelMixin): pass return None + def get_related_assets(self): + assets = self.assets.all() + return assets + def set_auth(self, password=None, private_key=None, public_key=None): update_fields = [] if password: @@ -124,17 +126,52 @@ class AssetUser(OrgModelMixin): if update_fields: self.save(update_fields=update_fields) - def get_auth(self, asset=None): - pass + def set_connectivity(self, summary): + unreachable = summary.get('dark', {}).keys() + reachable = summary.get('contacted', {}).keys() - def get_connectivity_of(self, asset): - i = self.generate_id_with_asset(asset) - key = self.CONNECTIVITY_CACHE_KEY.format(i) - return cache.get(key) + for asset in self.get_related_assets(): + if asset.hostname in unreachable: + self.set_asset_connectivity(asset, Connectivity.unreachable()) + elif asset.hostname in reachable: + self.set_asset_connectivity(asset, Connectivity.reachable()) + else: + self.set_asset_connectivity(asset, Connectivity.unknown()) - def set_connectivity_of(self, asset, c): + @property + def connectivity(self): + assets = self.get_related_assets() + data = { + 'unreachable': [], + 'reachable': [], + 'unknown': [], + } + for asset in assets: + connectivity = self.get_asset_connectivity(asset) + if connectivity.is_reachable(): + data["reachable"].append(asset.hostname) + elif connectivity.is_unreachable(): + data["unreachable"].append(asset.hostname) + else: + data["unknown"].append(asset.hostname) + return data + + @property + def connectivity_amount(self): + return {k: len(v) for k, v in self.connectivity.items()} + + @property + def assets_amount(self): + return self.get_related_assets().count() + + def get_asset_connectivity(self, asset): i = self.generate_id_with_asset(asset) - key = self.CONNECTIVITY_CACHE_KEY.format(i) + key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i) + return cache.get(key, const.CONN_UNKNOWN) + + def set_asset_connectivity(self, asset, c): + i = self.generate_id_with_asset(asset) + key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i) cache.set(key, c, 3600) def load_specific_asset_auth(self, asset): @@ -168,9 +205,10 @@ class AssetUser(OrgModelMixin): private_key, public_key = ssh_key_gen( username=self.username ) - self.set_auth(password=password, - private_key=private_key, - public_key=public_key) + self.set_auth( + password=password, private_key=private_key, + public_key=public_key + ) def auto_gen_auth_password(self): password = str(uuid.uuid4()) @@ -187,24 +225,18 @@ class AssetUser(OrgModelMixin): } def generate_id_with_asset(self, asset): - id_ = '{}_{}'.format(asset.id, self.id) - id_ = uuid.UUID(md5(id_.encode()).hexdigest()) - return id_ + i = '{}_{}'.format(asset.id, self.id) + i = uuid.UUID(md5(i.encode()).hexdigest()) + return i def construct_to_authbook(self, asset): - from . import AuthBook - fields = [ - 'name', 'username', 'comment', 'org_id', - '_password', '_private_key', '_public_key', - 'date_created', 'date_updated', 'created_by' - ] - id_ = self.generate_id_with_asset(asset) - obj = AuthBook(id=id_, asset=asset, version=0, is_latest=True) - obj._connectivity = self.get_connectivity_of(asset) - for field in fields: - value = getattr(self, field) - setattr(obj, field, value) - return obj + i = self.generate_id_with_asset(asset) + self.id = i + self.asset = asset + self.version = 0 + self.is_latest = True + return self class Meta: abstract = True + diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 2cded41a1..32f569b80 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -4,13 +4,11 @@ import logging -from django.core.cache import cache from django.db import models from django.utils.translation import ugettext_lazy as _ from django.core.validators import MinValueValidator, MaxValueValidator from common.utils import get_signer -from ..const import SYSTEM_USER_CONN_CACHE_KEY from .base import AssetUser @@ -31,7 +29,7 @@ class AdminUser(AssetUser): become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4) become_user = models.CharField(default='root', max_length=64) _become_pass = models.CharField(default='', max_length=128) - CONNECTIVE_CACHE_KEY = '_JMS_ADMIN_USER_CONNECTIVE_{}' + CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}' _prefer = "admin_user" def __str__(self): @@ -61,31 +59,6 @@ class AdminUser(AssetUser): info = None return info - def get_related_assets(self): - assets = self.assets.all() - return assets - - @property - def assets_amount(self): - return self.get_related_assets().count() - - @property - def connectivity(self): - from .asset import Asset - assets = self.get_related_assets().values_list('id', 'hostname', flat=True) - data = { - 'unreachable': [], - 'reachable': [], - } - for asset_id, hostname in assets: - key = Asset.CONNECTIVITY_CACHE_KEY.format(str(self.id)) - value = cache.get(key, Asset.UNKNOWN) - if value == Asset.REACHABLE: - data['reachable'].append(hostname) - elif value == Asset.UNREACHABLE: - data['unreachable'].append(hostname) - return data - class Meta: ordering = ['name'] unique_together = [('name', 'org_id')] @@ -141,9 +114,6 @@ class SystemUser(AssetUser): login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')) cmd_filters = models.ManyToManyField('CommandFilter', related_name='system_users', verbose_name=_("Command filter"), blank=True) - SYSTEM_USER_CACHE_KEY = "__SYSTEM_USER_CACHED_{}" - CONNECTIVE_CACHE_KEY = '_JMS_SYSTEM_USER_CONNECTIVE_{}' - def __str__(self): return '{0.name}({0.username})'.format(self) @@ -157,49 +127,6 @@ class SystemUser(AssetUser): 'auto_push': self.auto_push, } - def get_related_assets(self): - assets = set(self.assets.all()) - return assets - - @property - def connectivity(self): - cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) - value = cache.get(cache_key, None) - if not value or 'unreachable' not in value: - return {'unreachable': [], 'reachable': []} - else: - return value - - @connectivity.setter - def connectivity(self, value): - data = self.connectivity - unreachable = data['unreachable'] - reachable = data['reachable'] - assets = {asset.hostname: asset for asset in self.assets.all()} - - for host in value.get('dark', {}).keys(): - if host not in unreachable: - unreachable.append(host) - if host in reachable: - reachable.remove(host) - self.set_connectivity_of(assets.get(host), self.UNREACHABLE) - for host in value.get('contacted'): - if host not in reachable: - reachable.append(host) - if host in unreachable: - unreachable.remove(host) - self.set_connectivity_of(assets.get(host), self.REACHABLE) - cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) - cache.set(cache_key, data, 3600) - - @property - def assets_unreachable(self): - return self.connectivity.get('unreachable') - - @property - def assets_reachable(self): - return self.connectivity.get('reachable') - @property def login_mode_display(self): return self.get_login_mode_display() @@ -210,12 +137,6 @@ class SystemUser(AssetUser): else: return False - def set_cache(self): - cache.set(self.SYSTEM_USER_CACHE_KEY.format(self.id), self, 3600) - - def expire_cache(self): - cache.delete(self.SYSTEM_USER_CACHE_KEY.format(self.id)) - @property def cmd_filter_rules(self): from .cmd_filter import CommandFilterRule @@ -233,18 +154,6 @@ class SystemUser(AssetUser): return False, matched_cmd return True, None - @classmethod - def get_system_user_by_id_or_cached(cls, sid): - cached = cache.get(cls.SYSTEM_USER_CACHE_KEY.format(sid)) - if cached: - return cached - try: - system_user = cls.objects.get(id=sid) - system_user.set_cache() - return system_user - except cls.DoesNotExist: - return None - class Meta: ordering = ['name'] unique_together = [('name', 'org_id')] diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index 765559e70..142f5169a 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- # -from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from common.serializers import AdaptedBulkListSerializer from ..models import Node, AdminUser -from ..const import ADMIN_USER_CONN_CACHE_KEY from orgs.mixins import BulkOrgResourceModelSerializer from .base import AuthSerializer @@ -20,9 +18,6 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer): password = serializers.CharField( required=False, write_only=True, label=_('Password') ) - unreachable_amount = serializers.SerializerMethodField(label=_('Unreachable')) - assets_amount = serializers.SerializerMethodField(label=_('Asset')) - reachable_amount = serializers.SerializerMethodField(label=_('Reachable')) class Meta: list_serializer_class = AdaptedBulkListSerializer @@ -38,33 +33,14 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer): 'date_created': {'label': _('Date created')}, 'date_updated': {'label': _('Date updated')}, 'become': {'read_only': True}, 'become_method': {'read_only': True}, - 'become_user': {'read_only': True}, 'created_by': {'read_only': True} + 'become_user': {'read_only': True}, 'created_by': {'read_only': True}, + 'assets_amount': {'label', _('Asset')} } def get_field_names(self, declared_fields, info): fields = super().get_field_names(declared_fields, info) return [f for f in fields if not f.startswith('_')] - @staticmethod - def get_unreachable_amount(obj): - data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name)) - if data: - return len(data.get('dark')) - else: - return 0 - - @staticmethod - def get_reachable_amount(obj): - data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name)) - if data: - return len(data.get('contacted')) - else: - return 0 - - @staticmethod - def get_assets_amount(obj): - return obj.assets_amount - class AdminUserAuthSerializer(AuthSerializer): diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py index 149e9fa2c..01271be96 100644 --- a/apps/assets/tasks.py +++ b/apps/assets/tasks.py @@ -15,8 +15,9 @@ from ops.celery.decorator import ( register_as_period_task, after_app_shutdown_clean_periodic ) -from .models import SystemUser, AdminUser, Asset +from .models import SystemUser, AdminUser from . import const +from .utils import Connectivity FORKS = 10 @@ -208,7 +209,7 @@ def test_asset_connectivity_util(assets, task_name=None): created_by=created_by, ) result = task.run() - summary = result[1] + summary = result.get("summary", {}) success = summary.get('success', False) contacted = summary.get('contacted', {}) dark = summary.get('dark', {}) @@ -218,13 +219,12 @@ def test_asset_connectivity_util(assets, task_name=None): results_summary['dark'].update(dark) for asset in assets: - if asset.hostname in results_summary.get('dark', {}): - asset.connectivity = asset.UNREACHABLE - elif asset.hostname in results_summary.get('contacted', []): - asset.connectivity = asset.REACHABLE + if asset.hostname in results_summary.get('dark', {}).keys(): + asset.connectivity = Connectivity.unreachable() + elif asset.hostname in results_summary.get('contacted', {}).keys(): + asset.connectivity = Connectivity.reachable() else: - asset.connectivity = asset.UNKNOWN - + asset.connectivity = Connectivity.unknown() return results_summary @@ -286,10 +286,6 @@ def test_admin_user_connectivity_manual(admin_user): ## System user connective ## -@shared_task -def set_system_user_connectivity_info(system_user, summary): - system_user.connectivity = summary - @shared_task def test_system_user_connectivity_util(system_user, assets, task_name): @@ -346,7 +342,7 @@ def test_system_user_connectivity_util(system_user, assets, task_name): results_summary['contacted'].update(contacted) results_summary['dark'].update(dark) - set_system_user_connectivity_info(system_user, results_summary) + system_user.set_connectivity(results_summary) return results_summary @@ -584,6 +580,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False) """ :param asset_user: 对象 :param task_name: + :param run_as_admin: :return: """ from ops.utils import update_or_create_ansible_task @@ -607,7 +604,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False) kwargs["run_as"] = asset_user.username task, created = update_or_create_ansible_task(*args, **kwargs) result = task.run() - set_asset_user_connectivity_info(asset_user, result) + asset_user.set_connectivity(result.get("summary", {})) @shared_task diff --git a/apps/assets/utils.py b/apps/assets/utils.py index a4809e3a7..cd022a9aa 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -1,8 +1,8 @@ # ~*~ coding: utf-8 ~*~ # -import os -import paramiko -from paramiko.ssh_exception import SSHException +from django.utils.translation import ugettext_lazy as _ +from django.core.cache import cache +from django.utils import timezone from common.utils import get_object_or_none from .models import Asset, SystemUser, Label @@ -45,3 +45,62 @@ class LabelFilter: for kwargs in conditions: queryset = queryset.filter(**kwargs) return queryset + + +class Connectivity: + UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) + CONNECTIVITY_CHOICES = ( + (UNREACHABLE, _("Unreachable")), + (REACHABLE, _('Reachable')), + (UNKNOWN, _("Unknown")), + ) + + value = UNKNOWN + datetime = timezone.now() + + def __init__(self, value, datetime): + self.value = value + self.datetime = datetime + + def display(self): + return dict(self.__class__.CONNECTIVITY_CHOICES).get(self.value) + + def is_reachable(self): + return self.value == self.REACHABLE + + def is_unreachable(self): + return self.value == self.UNREACHABLE + + def is_unknown(self): + return self.value == self.UNKNOWN + + @classmethod + def unreachable(cls): + return cls(cls.UNREACHABLE, timezone.now()) + + @classmethod + def reachable(cls): + return cls(cls.REACHABLE, timezone.now()) + + @classmethod + def unknown(cls): + return cls(cls.UNKNOWN, timezone.now()) + + @classmethod + def set(cls, key, value, ttl=0): + cache.set(key, value, ttl) + + @classmethod + def get(cls, key): + return cache.get(key, cls.UNKNOWN) + + @classmethod + def set_unreachable(cls, key, ttl=0): + cls.set(key, cls.unreachable(), ttl) + + @classmethod + def set_reachable(cls, key, ttl=0): + cls.set(key, cls.reachable(), ttl) + + def __eq__(self, other): + return self.value == other.value diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 79146c039..6d79cfa6f 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -293,3 +293,11 @@ class LocalProxy(object): __rdivmod__ = lambda x, o: x._get_current_object().__rdivmod__(o) __copy__ = lambda x: copy.copy(x._get_current_object()) __deepcopy__ = lambda x, memo: copy.deepcopy(x._get_current_object(), memo) + + +def random_string(length): + import string + import random + charset = string.ascii_letters + string.digits + s = [random.choice(charset) for i in range(length)] + return ''.join(s) diff --git a/apps/terminal/api/session.py b/apps/terminal/api/session.py index 2943641a1..5d37be0e0 100644 --- a/apps/terminal/api/session.py +++ b/apps/terminal/api/session.py @@ -47,9 +47,8 @@ class SessionViewSet(BulkModelViewSet): sid = serializer.validated_data["system_user"] # guacamole提交的是id if is_uuid(sid): - _system_user = SystemUser.get_system_user_by_id_or_cached(sid) - if _system_user: - serializer.validated_data["system_user"] = _system_user.name + _system_user = get_object_or_404(SystemUser, id=sid) + serializer.validated_data["system_user"] = _system_user.name return super().perform_create(serializer)