perf: 优化migrations

apps/applications/models/application.py

@@ -9,7 +9,8 @@ from orgs.mixins.models import OrgModelMixin
 from common.mixins import CommonModelMixin
 from common.tree import TreeNode
 from common.utils import is_uuid
-from assets.models import Asset, SystemUser
+from assets.models import Asset
+from assets.models._user import SystemUser
 
 from ..utils import KubernetesTree
 from .. import const

apps/assets/migrations/0001_initial.py

@@ -16,14 +16,6 @@ def add_default_group(apps, schema_editor):
     )
 
 
-def add_default_cluster(apps, schema_editor):
-    cluster_model = apps.get_model("assets", "Cluster")
-    db_alias = schema_editor.connection.alias
-    cluster_model.objects.using(db_alias).create(
-        name="Default"
-    )
-
-
 class Migration(migrations.Migration):
 
     initial = True
@@ -163,6 +155,5 @@ class Migration(migrations.Migration):
             unique_together=set([('ip', 'port')]),
         ),
 
-        migrations.RunPython(add_default_cluster),
         migrations.RunPython(add_default_group),
     ]

apps/assets/migrations/0003_auto_20180109_2331.py

@@ -14,9 +14,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterField(
-            model_name='asset',
-            name='cluster',
-            field=models.ForeignKey(default=assets.models.asset.default_cluster, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='assets', to='assets.Cluster', verbose_name='Cluster'),
-        ),
     ]

apps/assets/migrations/0092_auto_20220711_1409.py

@@ -1,7 +1,6 @@
 # Generated by Django 3.2.12 on 2022-07-11 08:59
 
 import assets.models.base
-import assets.models.user
 import common.db.fields
 from django.conf import settings
 from django.db import migrations, models
@@ -36,7 +35,7 @@ class Migration(migrations.Migration):
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
                 ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
                 ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
-                ('version', models.IntegerField(default=1, verbose_name='Version')),
+                ('version', models.IntegerField(default=0, verbose_name='Version')),
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField(db_index=True)),
                 ('history_change_reason', models.CharField(max_length=100, null=True)),
@@ -78,6 +77,6 @@ class Migration(migrations.Migration):
                 'permissions': [('view_accountsecret', 'Can view asset account secret'), ('change_accountsecret', 'Can change asset account secret'), ('view_historyaccount', 'Can view asset history account'), ('view_historyaccountsecret', 'Can view asset history account secret')],
                 'unique_together': {('username', 'asset')},
             },
-            bases=(models.Model, assets.models.base.AuthMixin, assets.models.user.ProtocolMixin),
+            bases=(models.Model, assets.models.base.AuthMixin, assets.models.protocol.ProtocolMixin),
         ),
     ]

apps/assets/migrations/0094_auto_20220728_1125.py

@@ -1,89 +0,0 @@
-# Generated by Django 3.2.14 on 2022-07-28 03:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0093_auto_20220711_1413'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='cluster',
-            name='admin_user',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='ad_domain',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='assets',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='auto_push',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='groups',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='home',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='nodes',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='priority',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='sftp_root',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='shell',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='sudo',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='system_groups',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='token',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='type',
-        ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='users',
-        ),
-        migrations.AlterField(
-            model_name='historicalaccount',
-            name='version',
-            field=models.IntegerField(default=0, verbose_name='Version'),
-        ),
-        migrations.AlterField(
-            model_name='systemuser',
-            name='login_mode',
-            field=models.CharField(choices=[('auto', '使用账号'), ('manual', 'Manually input')], default='auto', max_length=10, verbose_name='Login mode'),
-        ),
-        migrations.DeleteModel(
-            name='AdminUser',
-        ),
-        migrations.DeleteModel(
-            name='Cluster',
-        ),
-    ]

apps/assets/migrations/0094_auto_20220803_1448.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.14 on 2022-08-03 06:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0093_auto_20220711_1413'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='Cluster',
+        ),
+    ]

apps/assets/models/__init__.py

@@ -1,4 +1,5 @@
 from .base import *
+from ._user import *
 from .asset import *
 from .label import Label
 from .group import *
@@ -9,5 +10,7 @@ from .gathered_user import *
 from .favorite_asset import *
 from .account import *
 from .backup import *
-from .user import *
+# 废弃以下
+from ._authbook import *
+from .protocol import *
 from .cmd_filter import *

apps/assets/models/_authbook.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+#
+
+from django.db import models
+from django.db.models import F
+from django.utils.translation import ugettext_lazy as _
+from simple_history.models import HistoricalRecords
+
+from common.utils import lazyproperty, get_logger
+from .base import BaseUser, AbsConnectivity
+
+logger = get_logger(__name__)
+
+
+__all__ = ['AuthBook']
+
+
+class AuthBook(BaseUser, AbsConnectivity):
+    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
+    systemuser = models.ForeignKey('assets.SystemUser', on_delete=models.CASCADE, null=True, verbose_name=_("System user"))
+    version = models.IntegerField(default=1, verbose_name=_('Version'))
+    history = HistoricalRecords()
+
+    auth_attrs = ['username', 'password', 'private_key', 'public_key']
+
+    class Meta:
+        verbose_name = _('AuthBook')
+        unique_together = [('username', 'asset', 'systemuser')]
+        permissions = [
+            ('test_authbook', _('Can test asset account connectivity')),
+            ('view_assetaccountsecret', _('Can view asset account secret')),
+            ('change_assetaccountsecret', _('Can change asset account secret')),
+            ('view_assethistoryaccount', _('Can view asset history account')),
+            ('view_assethistoryaccountsecret', _('Can view asset history account secret')),
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.auth_snapshot = {}
+
+    def get_or_systemuser_attr(self, attr):
+        val = getattr(self, attr, None)
+        if val:
+            return val
+        if self.systemuser:
+            return getattr(self.systemuser, attr, '')
+        return ''
+
+    def load_auth(self):
+        for attr in self.auth_attrs:
+            value = self.get_or_systemuser_attr(attr)
+            self.auth_snapshot[attr] = [getattr(self, attr), value]
+            setattr(self, attr, value)
+
+    def unload_auth(self):
+        if not self.systemuser:
+            return
+
+        for attr, values in self.auth_snapshot.items():
+            origin_value, loaded_value = values
+            current_value = getattr(self, attr, '')
+            if current_value == loaded_value:
+                setattr(self, attr, origin_value)
+
+    def save(self, *args, **kwargs):
+        self.unload_auth()
+        instance = super().save(*args, **kwargs)
+        self.load_auth()
+        return instance
+
+    @property
+    def username_display(self):
+        return self.get_or_systemuser_attr('username') or '*'
+
+    @lazyproperty
+    def systemuser_display(self):
+        if not self.systemuser:
+            return ''
+        return str(self.systemuser)
+
+    @property
+    def smart_name(self):
+        username = self.username_display
+
+        if self.asset:
+            asset = str(self.asset)
+        else:
+            asset = '*'
+        return '{}@{}'.format(username, asset)
+
+    def sync_to_system_user_account(self):
+        if self.systemuser:
+            return
+        matched = AuthBook.objects.filter(
+            asset=self.asset, systemuser__username=self.username
+        )
+        if not matched:
+            return
+
+        for i in matched:
+            i.password = self.password
+            i.private_key = self.private_key
+            i.public_key = self.public_key
+            i.comment = 'Update triggered by account {}'.format(self.id)
+
+        # 不触发post_save信号
+        self.__class__.objects.bulk_update(matched, fields=['password', 'private_key', 'public_key'])
+
+    def remove_asset_admin_user_if_need(self):
+        if not self.asset or not self.systemuser:
+            return
+        if not self.systemuser.is_admin_user or self.asset.admin_user != self.systemuser:
+            return
+        self.asset.admin_user = None
+        self.asset.save()
+        logger.debug('Remove asset admin user: {} {}'.format(self.asset, self.systemuser))
+
+    def update_asset_admin_user_if_need(self):
+        if not self.asset or not self.systemuser:
+            return
+        if not self.systemuser.is_admin_user or self.asset.admin_user == self.systemuser:
+            return
+        self.asset.admin_user = self.systemuser
+        self.asset.save()
+        logger.debug('Update asset admin user: {} {}'.format(self.asset, self.systemuser))
+
+    @classmethod
+    def get_queryset(cls):
+        queryset = cls.objects.all() \
+            .annotate(ip=F('asset__ip')) \
+            .annotate(hostname=F('asset__hostname')) \
+            .annotate(platform=F('asset__platform__name')) \
+            .annotate(protocols=F('asset__protocols'))
+        return queryset
+
+    def __str__(self):
+        return self.smart_name

apps/assets/models/_user.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+
+import logging
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from django.core.validators import MinValueValidator, MaxValueValidator
+
+from .base import BaseUser
+from .protocol import ProtocolMixin
+
+
+__all__ = ['SystemUser']
+logger = logging.getLogger(__name__)
+
+
+class SystemUser(ProtocolMixin, BaseUser):
+    LOGIN_AUTO = 'auto'
+    LOGIN_MANUAL = 'manual'
+    LOGIN_MODE_CHOICES = (
+        (LOGIN_AUTO, _('Automatic managed')),
+        (LOGIN_MANUAL, _('Manually input'))
+    )
+
+    class Type(models.TextChoices):
+        common = 'common', _('Common user')
+        admin = 'admin', _('Admin user')
+
+    username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user"))
+    nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes"))
+    assets = models.ManyToManyField(
+        'assets.Asset', blank=True, verbose_name=_("Assets"),
+        through='assets.AuthBook', through_fields=['systemuser', 'asset'],
+        related_name='system_users'
+    )
+    users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))
+    groups = models.ManyToManyField('users.UserGroup', blank=True, verbose_name=_("User groups"))
+    type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_('Type'))
+    priority = models.IntegerField(default=81, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), validators=[MinValueValidator(1), MaxValueValidator(100)])
+    protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol'))
+    auto_push = models.BooleanField(default=True, verbose_name=_('Auto push'))
+    sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo'))
+    shell = models.CharField(max_length=64,  default='/bin/bash', verbose_name=_('Shell'))
+    login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode'))
+    sftp_root = models.CharField(default='tmp', max_length=128, verbose_name=_("SFTP Root"))
+    token = models.TextField(default='', verbose_name=_('Token'))
+    home = models.CharField(max_length=4096, default='', verbose_name=_('Home'), blank=True)
+    system_groups = models.CharField(default='', max_length=4096, verbose_name=_('System groups'), blank=True)
+    ad_domain = models.CharField(default='', max_length=256)
+    # linux su 命令 (switch user)
+    su_enabled = models.BooleanField(default=False, verbose_name=_('User switch'))
+    su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from"))
+
+    def __str__(self):
+        username = self.username
+        if self.username_same_with_user:
+            username = '*'
+        return '{0.name}({1})'.format(self, username)
+
+    @property
+    def nodes_amount(self):
+        return self.nodes.all().count()
+
+    @property
+    def login_mode_display(self):
+        return self.get_login_mode_display()
+
+    def is_need_push(self):
+        if self.auto_push and self.is_protocol_support_push:
+            return True
+        else:
+            return False
+
+    @property
+    def is_admin_user(self):
+        return self.type == self.Type.admin
+
+    @property
+    def is_need_cmd_filter(self):
+        return self.protocol not in [self.Protocol.rdp, self.Protocol.vnc]
+
+    @property
+    def is_need_test_asset_connective(self):
+        return self.protocol in self.ASSET_CATEGORY_PROTOCOLS
+
+    @property
+    def cmd_filter_rules(self):
+        from .cmd_filter import CommandFilterRule
+        rules = CommandFilterRule.objects.filter(
+            filter__in=self.cmd_filters.all()
+        ).distinct()
+        return rules
+
+    def is_command_can_run(self, command):
+        for rule in self.cmd_filter_rules:
+            action, matched_cmd = rule.match(command)
+            if action == rule.ActionChoices.allow:
+                return True, None
+            elif action == rule.ActionChoices.deny:
+                return False, matched_cmd
+        return True, None
+
+    def get_all_assets(self):
+        from assets.models import Node, Asset
+        nodes_keys = self.nodes.all().values_list('key', flat=True)
+        asset_ids = set(self.assets.all().values_list('id', flat=True))
+        nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys)
+        asset_ids.update(nodes_asset_ids)
+        assets = Asset.objects.filter(id__in=asset_ids)
+        return assets
+
+    def add_related_assets(self, assets_or_ids):
+        self.assets.add(*tuple(assets_or_ids))
+        self.add_related_assets_to_su_from_if_need(assets_or_ids)
+
+    def add_related_assets_to_su_from_if_need(self, assets_or_ids):
+        if self.protocol not in [self.Protocol.ssh.value]:
+            return
+        if not self.su_enabled:
+            return
+        if not self.su_from:
+            return
+        if self.su_from.protocol != self.protocol:
+            return
+        self.su_from.assets.add(*tuple(assets_or_ids))
+
+    class Meta:
+        ordering = ['name']
+        unique_together = [('name', 'org_id')]
+        verbose_name = _("System user")
+        permissions = [
+            ('match_systemuser', _('Can match system user')),
+        ]
+
+
+# Deprecated: 准备废弃
+class AdminUser(BaseUser):
+    """
+    A privileged user that ansible can use it to push system user and so on
+    """
+    BECOME_METHOD_CHOICES = (
+        ('sudo', 'sudo'),
+        ('su', 'su'),
+    )
+    become = models.BooleanField(default=True)
+    become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
+    become_user = models.CharField(default='root', max_length=64)
+    _become_pass = models.CharField(default='', blank=True, max_length=128)
+    CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
+    _prefer = "admin_user"
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def become_pass(self):
+        password = signer.unsign(self._become_pass)
+        if password:
+            return password
+        else:
+            return ""
+
+    @become_pass.setter
+    def become_pass(self, password):
+        self._become_pass = signer.sign(password)
+
+    @property
+    def become_info(self):
+        if self.become:
+            info = {
+                "method": self.become_method,
+                "user": self.become_user,
+                "pass": self.become_pass,
+            }
+        else:
+            info = None
+        return info
+
+    class Meta:
+        ordering = ['name']
+        unique_together = [('name', 'org_id')]
+        verbose_name = _("Admin user")

apps/assets/models/asset.py

@@ -9,7 +9,6 @@ from collections import OrderedDict
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from rest_framework.exceptions import ValidationError
 
 from common.db.fields import JsonDictTextField
 from common.utils import lazyproperty
@@ -21,16 +20,6 @@ __all__ = ['Asset', 'ProtocolsMixin', 'Platform', 'AssetQuerySet']
 logger = logging.getLogger(__name__)
 
 
-def default_cluster():
-    from .cluster import Cluster
-    name = "Default"
-    defaults = {"name": name}
-    cluster, created = Cluster.objects.get_or_create(
-        defaults=defaults, name=name
-    )
-    return cluster.id
-
-
 def default_node():
     try:
         from .node import Node

apps/assets/models/cmd_filter.py

@@ -9,7 +9,6 @@ from django.core.validators import MinValueValidator, MaxValueValidator
 from django.utils.translation import ugettext_lazy as _
 
 from users.models import User, UserGroup
-from applications.models import Application
 from ..models import SystemUser, Asset
 
 from common.utils import lazyproperty, get_logger, get_object_or_none
@@ -190,6 +189,7 @@ class CommandFilterRule(OrgModelMixin):
     @classmethod
     def get_queryset(cls, user_id=None, user_group_id=None, system_user_id=None,
                      asset_id=None, application_id=None, org_id=None):
+        from applications.models import Application
         user_groups = []
         user = get_object_or_none(User, pk=user_id)
         if user:

apps/assets/models/user.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-
-import logging
-
-from django.db import models
-from django.utils.translation import ugettext_lazy as _
-from django.shortcuts import get_object_or_404
-from django.core.cache import cache
-
-from .base import BaseUser
-from .protocol import ProtocolMixin
-
-
-__all__ = ['SystemUser']
-logger = logging.getLogger(__name__)
-
-
-class SystemUser(ProtocolMixin, BaseUser):
-    LOGIN_AUTO = 'auto'
-    LOGIN_MANUAL = 'manual'
-    LOGIN_MODE_CHOICES = (
-        (LOGIN_AUTO, _('使用账号')),
-        (LOGIN_MANUAL, _('Manually input'))
-    )
-
-    username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user"))
-    protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol'))
-    login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode'))
-
-    # linux su 命令 (switch user)
-    # Todo: 修改为 username, 不必系统用户了
-    su_enabled = models.BooleanField(default=False, verbose_name=_('User switch'))
-    su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from"))
-
-    def __str__(self):
-        username = self.username
-        if self.username_same_with_user:
-            username = '*'
-        return '{0.name}({1})'.format(self, username)
-
-    @classmethod
-    def create_accounts_with_assets(cls, asset_ids, system_user_ids):
-        pass
-
-    def get_manual_account(self, user_id, asset_id):
-        cache_key = 'manual_account_{}_{}_{}'.format(self.id, user_id, asset_id)
-        return cache.get(cache_key)
-
-    def create_manual_account(self, user_id, asset_id, account, ttl=300):
-        cache_key = 'manual_account_{}_{}_{}'.format(self.id, user_id, asset_id)
-        cache.set(cache_key, account, ttl)
-
-    def get_auto_account(self, user_id, asset_id):
-        from .account import Account
-        from users.models import User
-        username = self.username
-        if self.username_same_with_user:
-            user = get_object_or_404(User, id=user_id)
-            username = user.username
-        return get_object_or_404(Account, asset_id=asset_id, username=username)
-
-    def get_account(self, user_id, asset_id):
-        if self.login_mode == self.LOGIN_MANUAL:
-            return self.get_manual_account(user_id, asset_id)
-        else:
-            return self.get_auto_account(user_id, asset_id)
-
-    class Meta:
-        ordering = ['name']
-        unique_together = [('name', 'org_id')]
-        verbose_name = _("System user")
-        permissions = [
-            ('match_systemuser', _('Can match system user')),
-        ]