From 65423ea893b0b8cd4364355a0ef4ddd0ce1984c1 Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 3 Aug 2022 15:58:06 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96migrations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/applications/models/application.py | 3 +- apps/assets/migrations/0001_initial.py | 9 - .../migrations/0003_auto_20180109_2331.py | 5 - .../migrations/0092_auto_20220711_1409.py | 5 +- .../migrations/0094_auto_20220728_1125.py | 89 --------- .../migrations/0094_auto_20220803_1448.py | 16 ++ apps/assets/models/__init__.py | 5 +- apps/assets/models/_authbook.py | 137 +++++++++++++ apps/assets/models/_user.py | 184 ++++++++++++++++++ apps/assets/models/asset.py | 11 -- apps/assets/models/cmd_filter.py | 2 +- apps/assets/models/user.py | 76 -------- 12 files changed, 346 insertions(+), 196 deletions(-) delete mode 100644 apps/assets/migrations/0094_auto_20220728_1125.py create mode 100644 apps/assets/migrations/0094_auto_20220803_1448.py create mode 100644 apps/assets/models/_authbook.py create mode 100644 apps/assets/models/_user.py delete mode 100644 apps/assets/models/user.py diff --git a/apps/applications/models/application.py b/apps/applications/models/application.py index af1e27c2d..258f2fe6e 100644 --- a/apps/applications/models/application.py +++ b/apps/applications/models/application.py @@ -9,7 +9,8 @@ from orgs.mixins.models import OrgModelMixin from common.mixins import CommonModelMixin from common.tree import TreeNode from common.utils import is_uuid -from assets.models import Asset, SystemUser +from assets.models import Asset +from assets.models._user import SystemUser from ..utils import KubernetesTree from .. import const diff --git a/apps/assets/migrations/0001_initial.py b/apps/assets/migrations/0001_initial.py index 7c0a9e95a..03c945c57 100644 --- a/apps/assets/migrations/0001_initial.py +++ b/apps/assets/migrations/0001_initial.py @@ -16,14 +16,6 @@ def add_default_group(apps, schema_editor): ) -def add_default_cluster(apps, schema_editor): - cluster_model = apps.get_model("assets", "Cluster") - db_alias = schema_editor.connection.alias - cluster_model.objects.using(db_alias).create( - name="Default" - ) - - class Migration(migrations.Migration): initial = True @@ -163,6 +155,5 @@ class Migration(migrations.Migration): unique_together=set([('ip', 'port')]), ), - migrations.RunPython(add_default_cluster), migrations.RunPython(add_default_group), ] diff --git a/apps/assets/migrations/0003_auto_20180109_2331.py b/apps/assets/migrations/0003_auto_20180109_2331.py index 254de6236..097bc607a 100644 --- a/apps/assets/migrations/0003_auto_20180109_2331.py +++ b/apps/assets/migrations/0003_auto_20180109_2331.py @@ -14,9 +14,4 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name='asset', - name='cluster', - field=models.ForeignKey(default=assets.models.asset.default_cluster, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='assets', to='assets.Cluster', verbose_name='Cluster'), - ), ] diff --git a/apps/assets/migrations/0092_auto_20220711_1409.py b/apps/assets/migrations/0092_auto_20220711_1409.py index efcb59e9d..6515392a0 100644 --- a/apps/assets/migrations/0092_auto_20220711_1409.py +++ b/apps/assets/migrations/0092_auto_20220711_1409.py @@ -1,7 +1,6 @@ # Generated by Django 3.2.12 on 2022-07-11 08:59 import assets.models.base -import assets.models.user import common.db.fields from django.conf import settings from django.db import migrations, models @@ -36,7 +35,7 @@ class Migration(migrations.Migration): ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')), ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')), ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')), - ('version', models.IntegerField(default=1, verbose_name='Version')), + ('version', models.IntegerField(default=0, verbose_name='Version')), ('history_id', models.AutoField(primary_key=True, serialize=False)), ('history_date', models.DateTimeField(db_index=True)), ('history_change_reason', models.CharField(max_length=100, null=True)), @@ -78,6 +77,6 @@ class Migration(migrations.Migration): 'permissions': [('view_accountsecret', 'Can view asset account secret'), ('change_accountsecret', 'Can change asset account secret'), ('view_historyaccount', 'Can view asset history account'), ('view_historyaccountsecret', 'Can view asset history account secret')], 'unique_together': {('username', 'asset')}, }, - bases=(models.Model, assets.models.base.AuthMixin, assets.models.user.ProtocolMixin), + bases=(models.Model, assets.models.base.AuthMixin, assets.models.protocol.ProtocolMixin), ), ] diff --git a/apps/assets/migrations/0094_auto_20220728_1125.py b/apps/assets/migrations/0094_auto_20220728_1125.py deleted file mode 100644 index adc477c1c..000000000 --- a/apps/assets/migrations/0094_auto_20220728_1125.py +++ /dev/null @@ -1,89 +0,0 @@ -# Generated by Django 3.2.14 on 2022-07-28 03:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('assets', '0093_auto_20220711_1413'), - ] - - operations = [ - migrations.RemoveField( - model_name='cluster', - name='admin_user', - ), - migrations.RemoveField( - model_name='systemuser', - name='ad_domain', - ), - migrations.RemoveField( - model_name='systemuser', - name='assets', - ), - migrations.RemoveField( - model_name='systemuser', - name='auto_push', - ), - migrations.RemoveField( - model_name='systemuser', - name='groups', - ), - migrations.RemoveField( - model_name='systemuser', - name='home', - ), - migrations.RemoveField( - model_name='systemuser', - name='nodes', - ), - migrations.RemoveField( - model_name='systemuser', - name='priority', - ), - migrations.RemoveField( - model_name='systemuser', - name='sftp_root', - ), - migrations.RemoveField( - model_name='systemuser', - name='shell', - ), - migrations.RemoveField( - model_name='systemuser', - name='sudo', - ), - migrations.RemoveField( - model_name='systemuser', - name='system_groups', - ), - migrations.RemoveField( - model_name='systemuser', - name='token', - ), - migrations.RemoveField( - model_name='systemuser', - name='type', - ), - migrations.RemoveField( - model_name='systemuser', - name='users', - ), - migrations.AlterField( - model_name='historicalaccount', - name='version', - field=models.IntegerField(default=0, verbose_name='Version'), - ), - migrations.AlterField( - model_name='systemuser', - name='login_mode', - field=models.CharField(choices=[('auto', '使用账号'), ('manual', 'Manually input')], default='auto', max_length=10, verbose_name='Login mode'), - ), - migrations.DeleteModel( - name='AdminUser', - ), - migrations.DeleteModel( - name='Cluster', - ), - ] diff --git a/apps/assets/migrations/0094_auto_20220803_1448.py b/apps/assets/migrations/0094_auto_20220803_1448.py new file mode 100644 index 000000000..eafb9caca --- /dev/null +++ b/apps/assets/migrations/0094_auto_20220803_1448.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.14 on 2022-08-03 06:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0093_auto_20220711_1413'), + ] + + operations = [ + migrations.DeleteModel( + name='Cluster', + ), + ] diff --git a/apps/assets/models/__init__.py b/apps/assets/models/__init__.py index 7719ca63a..e433a95ef 100644 --- a/apps/assets/models/__init__.py +++ b/apps/assets/models/__init__.py @@ -1,4 +1,5 @@ from .base import * +from ._user import * from .asset import * from .label import Label from .group import * @@ -9,5 +10,7 @@ from .gathered_user import * from .favorite_asset import * from .account import * from .backup import * -from .user import * +# 废弃以下 +from ._authbook import * +from .protocol import * from .cmd_filter import * diff --git a/apps/assets/models/_authbook.py b/apps/assets/models/_authbook.py new file mode 100644 index 000000000..e96196d22 --- /dev/null +++ b/apps/assets/models/_authbook.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# + +from django.db import models +from django.db.models import F +from django.utils.translation import ugettext_lazy as _ +from simple_history.models import HistoricalRecords + +from common.utils import lazyproperty, get_logger +from .base import BaseUser, AbsConnectivity + +logger = get_logger(__name__) + + +__all__ = ['AuthBook'] + + +class AuthBook(BaseUser, AbsConnectivity): + asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset')) + systemuser = models.ForeignKey('assets.SystemUser', on_delete=models.CASCADE, null=True, verbose_name=_("System user")) + version = models.IntegerField(default=1, verbose_name=_('Version')) + history = HistoricalRecords() + + auth_attrs = ['username', 'password', 'private_key', 'public_key'] + + class Meta: + verbose_name = _('AuthBook') + unique_together = [('username', 'asset', 'systemuser')] + permissions = [ + ('test_authbook', _('Can test asset account connectivity')), + ('view_assetaccountsecret', _('Can view asset account secret')), + ('change_assetaccountsecret', _('Can change asset account secret')), + ('view_assethistoryaccount', _('Can view asset history account')), + ('view_assethistoryaccountsecret', _('Can view asset history account secret')), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.auth_snapshot = {} + + def get_or_systemuser_attr(self, attr): + val = getattr(self, attr, None) + if val: + return val + if self.systemuser: + return getattr(self.systemuser, attr, '') + return '' + + def load_auth(self): + for attr in self.auth_attrs: + value = self.get_or_systemuser_attr(attr) + self.auth_snapshot[attr] = [getattr(self, attr), value] + setattr(self, attr, value) + + def unload_auth(self): + if not self.systemuser: + return + + for attr, values in self.auth_snapshot.items(): + origin_value, loaded_value = values + current_value = getattr(self, attr, '') + if current_value == loaded_value: + setattr(self, attr, origin_value) + + def save(self, *args, **kwargs): + self.unload_auth() + instance = super().save(*args, **kwargs) + self.load_auth() + return instance + + @property + def username_display(self): + return self.get_or_systemuser_attr('username') or '*' + + @lazyproperty + def systemuser_display(self): + if not self.systemuser: + return '' + return str(self.systemuser) + + @property + def smart_name(self): + username = self.username_display + + if self.asset: + asset = str(self.asset) + else: + asset = '*' + return '{}@{}'.format(username, asset) + + def sync_to_system_user_account(self): + if self.systemuser: + return + matched = AuthBook.objects.filter( + asset=self.asset, systemuser__username=self.username + ) + if not matched: + return + + for i in matched: + i.password = self.password + i.private_key = self.private_key + i.public_key = self.public_key + i.comment = 'Update triggered by account {}'.format(self.id) + + # 不触发post_save信号 + self.__class__.objects.bulk_update(matched, fields=['password', 'private_key', 'public_key']) + + def remove_asset_admin_user_if_need(self): + if not self.asset or not self.systemuser: + return + if not self.systemuser.is_admin_user or self.asset.admin_user != self.systemuser: + return + self.asset.admin_user = None + self.asset.save() + logger.debug('Remove asset admin user: {} {}'.format(self.asset, self.systemuser)) + + def update_asset_admin_user_if_need(self): + if not self.asset or not self.systemuser: + return + if not self.systemuser.is_admin_user or self.asset.admin_user == self.systemuser: + return + self.asset.admin_user = self.systemuser + self.asset.save() + logger.debug('Update asset admin user: {} {}'.format(self.asset, self.systemuser)) + + @classmethod + def get_queryset(cls): + queryset = cls.objects.all() \ + .annotate(ip=F('asset__ip')) \ + .annotate(hostname=F('asset__hostname')) \ + .annotate(platform=F('asset__platform__name')) \ + .annotate(protocols=F('asset__protocols')) + return queryset + + def __str__(self): + return self.smart_name diff --git a/apps/assets/models/_user.py b/apps/assets/models/_user.py new file mode 100644 index 000000000..565513c11 --- /dev/null +++ b/apps/assets/models/_user.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +import logging + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from django.core.validators import MinValueValidator, MaxValueValidator + +from .base import BaseUser +from .protocol import ProtocolMixin + + +__all__ = ['SystemUser'] +logger = logging.getLogger(__name__) + + +class SystemUser(ProtocolMixin, BaseUser): + LOGIN_AUTO = 'auto' + LOGIN_MANUAL = 'manual' + LOGIN_MODE_CHOICES = ( + (LOGIN_AUTO, _('Automatic managed')), + (LOGIN_MANUAL, _('Manually input')) + ) + + class Type(models.TextChoices): + common = 'common', _('Common user') + admin = 'admin', _('Admin user') + + username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user")) + nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes")) + assets = models.ManyToManyField( + 'assets.Asset', blank=True, verbose_name=_("Assets"), + through='assets.AuthBook', through_fields=['systemuser', 'asset'], + related_name='system_users' + ) + users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users")) + groups = models.ManyToManyField('users.UserGroup', blank=True, verbose_name=_("User groups")) + type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_('Type')) + priority = models.IntegerField(default=81, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), validators=[MinValueValidator(1), MaxValueValidator(100)]) + protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol')) + auto_push = models.BooleanField(default=True, verbose_name=_('Auto push')) + sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo')) + shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell')) + login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')) + sftp_root = models.CharField(default='tmp', max_length=128, verbose_name=_("SFTP Root")) + token = models.TextField(default='', verbose_name=_('Token')) + home = models.CharField(max_length=4096, default='', verbose_name=_('Home'), blank=True) + system_groups = models.CharField(default='', max_length=4096, verbose_name=_('System groups'), blank=True) + ad_domain = models.CharField(default='', max_length=256) + # linux su 命令 (switch user) + su_enabled = models.BooleanField(default=False, verbose_name=_('User switch')) + su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from")) + + def __str__(self): + username = self.username + if self.username_same_with_user: + username = '*' + return '{0.name}({1})'.format(self, username) + + @property + def nodes_amount(self): + return self.nodes.all().count() + + @property + def login_mode_display(self): + return self.get_login_mode_display() + + def is_need_push(self): + if self.auto_push and self.is_protocol_support_push: + return True + else: + return False + + @property + def is_admin_user(self): + return self.type == self.Type.admin + + @property + def is_need_cmd_filter(self): + return self.protocol not in [self.Protocol.rdp, self.Protocol.vnc] + + @property + def is_need_test_asset_connective(self): + return self.protocol in self.ASSET_CATEGORY_PROTOCOLS + + @property + def cmd_filter_rules(self): + from .cmd_filter import CommandFilterRule + rules = CommandFilterRule.objects.filter( + filter__in=self.cmd_filters.all() + ).distinct() + return rules + + def is_command_can_run(self, command): + for rule in self.cmd_filter_rules: + action, matched_cmd = rule.match(command) + if action == rule.ActionChoices.allow: + return True, None + elif action == rule.ActionChoices.deny: + return False, matched_cmd + return True, None + + def get_all_assets(self): + from assets.models import Node, Asset + nodes_keys = self.nodes.all().values_list('key', flat=True) + asset_ids = set(self.assets.all().values_list('id', flat=True)) + nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys) + asset_ids.update(nodes_asset_ids) + assets = Asset.objects.filter(id__in=asset_ids) + return assets + + def add_related_assets(self, assets_or_ids): + self.assets.add(*tuple(assets_or_ids)) + self.add_related_assets_to_su_from_if_need(assets_or_ids) + + def add_related_assets_to_su_from_if_need(self, assets_or_ids): + if self.protocol not in [self.Protocol.ssh.value]: + return + if not self.su_enabled: + return + if not self.su_from: + return + if self.su_from.protocol != self.protocol: + return + self.su_from.assets.add(*tuple(assets_or_ids)) + + class Meta: + ordering = ['name'] + unique_together = [('name', 'org_id')] + verbose_name = _("System user") + permissions = [ + ('match_systemuser', _('Can match system user')), + ] + + +# Deprecated: 准备废弃 +class AdminUser(BaseUser): + """ + A privileged user that ansible can use it to push system user and so on + """ + BECOME_METHOD_CHOICES = ( + ('sudo', 'sudo'), + ('su', 'su'), + ) + become = models.BooleanField(default=True) + become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4) + become_user = models.CharField(default='root', max_length=64) + _become_pass = models.CharField(default='', blank=True, max_length=128) + CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}' + _prefer = "admin_user" + + def __str__(self): + return self.name + + @property + def become_pass(self): + password = signer.unsign(self._become_pass) + if password: + return password + else: + return "" + + @become_pass.setter + def become_pass(self, password): + self._become_pass = signer.sign(password) + + @property + def become_info(self): + if self.become: + info = { + "method": self.become_method, + "user": self.become_user, + "pass": self.become_pass, + } + else: + info = None + return info + + class Meta: + ordering = ['name'] + unique_together = [('name', 'org_id')] + verbose_name = _("Admin user") diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 157895dae..1eedc21dd 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -9,7 +9,6 @@ from collections import OrderedDict from django.db import models from django.utils.translation import ugettext_lazy as _ -from rest_framework.exceptions import ValidationError from common.db.fields import JsonDictTextField from common.utils import lazyproperty @@ -21,16 +20,6 @@ __all__ = ['Asset', 'ProtocolsMixin', 'Platform', 'AssetQuerySet'] logger = logging.getLogger(__name__) -def default_cluster(): - from .cluster import Cluster - name = "Default" - defaults = {"name": name} - cluster, created = Cluster.objects.get_or_create( - defaults=defaults, name=name - ) - return cluster.id - - def default_node(): try: from .node import Node diff --git a/apps/assets/models/cmd_filter.py b/apps/assets/models/cmd_filter.py index c7fa33aae..86e613d3d 100644 --- a/apps/assets/models/cmd_filter.py +++ b/apps/assets/models/cmd_filter.py @@ -9,7 +9,6 @@ from django.core.validators import MinValueValidator, MaxValueValidator from django.utils.translation import ugettext_lazy as _ from users.models import User, UserGroup -from applications.models import Application from ..models import SystemUser, Asset from common.utils import lazyproperty, get_logger, get_object_or_none @@ -190,6 +189,7 @@ class CommandFilterRule(OrgModelMixin): @classmethod def get_queryset(cls, user_id=None, user_group_id=None, system_user_id=None, asset_id=None, application_id=None, org_id=None): + from applications.models import Application user_groups = [] user = get_object_or_none(User, pk=user_id) if user: diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py deleted file mode 100644 index 79d6e96e2..000000000 --- a/apps/assets/models/user.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# - -import logging - -from django.db import models -from django.utils.translation import ugettext_lazy as _ -from django.shortcuts import get_object_or_404 -from django.core.cache import cache - -from .base import BaseUser -from .protocol import ProtocolMixin - - -__all__ = ['SystemUser'] -logger = logging.getLogger(__name__) - - -class SystemUser(ProtocolMixin, BaseUser): - LOGIN_AUTO = 'auto' - LOGIN_MANUAL = 'manual' - LOGIN_MODE_CHOICES = ( - (LOGIN_AUTO, _('使用账号')), - (LOGIN_MANUAL, _('Manually input')) - ) - - username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user")) - protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol')) - login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')) - - # linux su 命令 (switch user) - # Todo: 修改为 username, 不必系统用户了 - su_enabled = models.BooleanField(default=False, verbose_name=_('User switch')) - su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from")) - - def __str__(self): - username = self.username - if self.username_same_with_user: - username = '*' - return '{0.name}({1})'.format(self, username) - - @classmethod - def create_accounts_with_assets(cls, asset_ids, system_user_ids): - pass - - def get_manual_account(self, user_id, asset_id): - cache_key = 'manual_account_{}_{}_{}'.format(self.id, user_id, asset_id) - return cache.get(cache_key) - - def create_manual_account(self, user_id, asset_id, account, ttl=300): - cache_key = 'manual_account_{}_{}_{}'.format(self.id, user_id, asset_id) - cache.set(cache_key, account, ttl) - - def get_auto_account(self, user_id, asset_id): - from .account import Account - from users.models import User - username = self.username - if self.username_same_with_user: - user = get_object_or_404(User, id=user_id) - username = user.username - return get_object_or_404(Account, asset_id=asset_id, username=username) - - def get_account(self, user_id, asset_id): - if self.login_mode == self.LOGIN_MANUAL: - return self.get_manual_account(user_id, asset_id) - else: - return self.get_auto_account(user_id, asset_id) - - class Meta: - ordering = ['name'] - unique_together = [('name', 'org_id')] - verbose_name = _("System user") - permissions = [ - ('match_systemuser', _('Can match system user')), - ]