From 12b74093e29f312acea696c210974be3fb6ea9d8 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 28 Oct 2022 18:19:44 +0800 Subject: [PATCH] =?UTF-8?q?pref:=20=E4=BF=AE=E6=94=B9=20applet=20host=20de?= =?UTF-8?q?ploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/ansible/inventory.py | 10 ++--- apps/ops/api/celery.py | 2 +- apps/terminal/api/applet/applet.py | 14 +++--- apps/terminal/api/applet/host.py | 24 ++++++++--- .../automations/deploy_applet_host/manager.py | 43 +++++++++++++++++++ .../deploy_applet_host/playbook.yml | 2 +- .../migrations/0055_auto_20221028_1544.py | 42 ++++++++++++++++++ apps/terminal/models/applet/applet.py | 5 ++- apps/terminal/models/applet/host.py | 22 ++++------ apps/terminal/serializers/applet.py | 15 +------ apps/terminal/tasks.py | 8 +++- apps/terminal/urls/api_urls.py | 7 +-- 12 files changed, 140 insertions(+), 54 deletions(-) create mode 100644 apps/terminal/migrations/0055_auto_20221028_1544.py diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 919b0948c..aee85face 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -9,8 +9,9 @@ __all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, manager, assets=None, account_policy='smart', - account_prefer='root,administrator', host_callback=None): + def __init__(self, assets, account_policy='smart', + account_prefer='root,administrator', + host_callback=None): """ :param assets: :param account_prefer: account username name if not set use account_policy @@ -79,10 +80,7 @@ class JMSInventory: ssh_protocol_matched = list(filter(lambda x: x.name == 'ssh', protocols)) ssh_protocol = ssh_protocol_matched[0] if ssh_protocol_matched else None host['ansible_host'] = asset.address - if asset.port == 0: - host['ansible_port'] = ssh_protocol.port if ssh_protocol else 22 - else: - host['ansible_port'] = asset.port + host['ansible_port'] = ssh_protocol.port if ssh_protocol else 22 su_from = account.su_from if platform.su_enabled and su_from: diff --git a/apps/ops/api/celery.py b/apps/ops/api/celery.py index 85a5c00d2..61d13db17 100644 --- a/apps/ops/api/celery.py +++ b/apps/ops/api/celery.py @@ -99,7 +99,7 @@ class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet): class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): - queryset = CeleryTask.objects.filter(name__in=['ops.tasks.hello', 'ops.tasks.hello_error', 'ops.tasks.hello_random']) + queryset = CeleryTask.objects.all() serializer_class = CeleryTaskSerializer http_method_names = ('get', 'head', 'options',) diff --git a/apps/terminal/api/applet/applet.py b/apps/terminal/api/applet/applet.py index 9ded63dc1..bce0738d6 100644 --- a/apps/terminal/api/applet/applet.py +++ b/apps/terminal/api/applet/applet.py @@ -1,7 +1,7 @@ -import os.path import shutil import zipfile import yaml +import os.path from django.core.files.storage import default_storage from rest_framework import viewsets @@ -9,12 +9,16 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.serializers import ValidationError -from terminal import serializers, models +from terminal import serializers +from terminal.models import AppletPublication, Applet from terminal.serializers import AppletUploadSerializer +__all__ = ['AppletViewSet', 'AppletPublicationViewSet'] + + class AppletViewSet(viewsets.ModelViewSet): - queryset = models.Applet.objects.all() + queryset = Applet.objects.all() serializer_class = serializers.AppletSerializer rbac_perms = { 'upload': 'terminal.add_applet', @@ -67,7 +71,7 @@ class AppletViewSet(viewsets.ModelViewSet): name = manifest['name'] update = request.query_params.get('update') - instance = models.Applet.objects.filter(name=name).first() + instance = Applet.objects.filter(name=name).first() if instance and not update: return Response({'error': 'Applet already exists: {}'.format(name)}, status=400) @@ -82,5 +86,5 @@ class AppletViewSet(viewsets.ModelViewSet): class AppletPublicationViewSet(viewsets.ModelViewSet): - queryset = models.AppletPublication.objects.all() + queryset = AppletPublication.objects.all() serializer_class = serializers.AppletPublicationSerializer diff --git a/apps/terminal/api/applet/host.py b/apps/terminal/api/applet/host.py index d4166ea98..ea8831b8f 100644 --- a/apps/terminal/api/applet/host.py +++ b/apps/terminal/api/applet/host.py @@ -1,23 +1,35 @@ from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.response import Response from orgs.utils import tmp_to_builtin_org -from terminal import serializers, models +from terminal import serializers +from terminal.models import AppletHost, Applet +from terminal.tasks import run_applet_host_deployment -__all__ = ['AppletHostViewSet', 'AppletHostDeploymentViewSet'] +__all__ = ['AppletHostViewSet'] class AppletHostViewSet(viewsets.ModelViewSet): serializer_class = serializers.AppletHostSerializer def get_queryset(self): - return models.AppletHost.objects.all() + return AppletHost.objects.all() def dispatch(self, request, *args, **kwargs): with tmp_to_builtin_org(system=1): return super().dispatch(request, *args, **kwargs) + @action(methods=['post'], detail=True) + def deploy(self, request): + from terminal.automations.deploy_applet_host.manager import DeployAppletHostManager + manager = DeployAppletHostManager(self) + manager.run() -class AppletHostDeploymentViewSet(viewsets.ModelViewSet): - queryset = models.AppletHostDeployment.objects.all() - serializer_class = serializers.AppletHostDeploymentSerializer + @action(methods=['get'], detail=True, url_path='') + def not_published_applets(self, request, *args, **kwargs): + instance = self.get_object() + applets = Applet.objects.exclude(id__in=instance.applets.all()) + serializer = serializers.AppletSerializer(applets, many=True) + return Response(serializer.data) diff --git a/apps/terminal/automations/deploy_applet_host/manager.py b/apps/terminal/automations/deploy_applet_host/manager.py index e69de29bb..22a8510af 100644 --- a/apps/terminal/automations/deploy_applet_host/manager.py +++ b/apps/terminal/automations/deploy_applet_host/manager.py @@ -0,0 +1,43 @@ +import os +import datetime +import shutil +from django.conf import settings + +from ops.ansible import PlaybookRunner, JMSInventory + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class DeployAppletHostManager: + def __init__(self, applet_host): + self.applet_host = applet_host + self.run_dir = self.get_run_dir() + + @staticmethod + def get_run_dir(): + base = os.path.join(settings.ANSIBLE_DIR, 'applet_host_deploy') + now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + return os.path.join(base, now) + + def generate_playbook(self): + playbook_src = os.path.join(CURRENT_DIR, 'playbook.yml') + playbook_dir = os.path.join(self.run_dir, 'playbook') + playbook_dst = os.path.join(playbook_dir, 'main.yml') + os.makedirs(playbook_dir, exist_ok=True) + shutil.copy(playbook_src, playbook_dst) + return playbook_dst + + def generate_inventory(self): + inventory = JMSInventory([self.applet_host], account_policy='privileged_only') + inventory_dir = os.path.join(self.run_dir, 'inventory') + inventory_path = os.path.join(inventory_dir, 'hosts.yml') + inventory.write_to_file(inventory_path) + return inventory_path + + def run(self, **kwargs): + inventory = self.generate_inventory() + playbook = self.generate_playbook() + runner = PlaybookRunner( + inventory=inventory, playbook=playbook, project_dir=self.run_dir + ) + return runner.run(**kwargs) diff --git a/apps/terminal/automations/deploy_applet_host/playbook.yml b/apps/terminal/automations/deploy_applet_host/playbook.yml index 443482fc1..348f50fdb 100644 --- a/apps/terminal/automations/deploy_applet_host/playbook.yml +++ b/apps/terminal/automations/deploy_applet_host/playbook.yml @@ -1,6 +1,6 @@ --- -- hosts: windows +- hosts: all vars: - DownloadHost: https://demo.jumpserver.org/download - RDS_Licensing: enabled diff --git a/apps/terminal/migrations/0055_auto_20221028_1544.py b/apps/terminal/migrations/0055_auto_20221028_1544.py new file mode 100644 index 000000000..3aeeff3f6 --- /dev/null +++ b/apps/terminal/migrations/0055_auto_20221028_1544.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.14 on 2022-10-28 07:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('terminal', '0054_auto_20221027_1125'), + ] + + operations = [ + migrations.AddField( + model_name='applet', + name='hosts', + field=models.ManyToManyField(through='terminal.AppletPublication', to='terminal.AppletHost', verbose_name='Hosts'), + ), + migrations.AddField( + model_name='applethost', + name='date_inited', + field=models.DateTimeField(blank=True, null=True, verbose_name='Date initialized'), + ), + migrations.AddField( + model_name='applethost', + name='initialized', + field=models.BooleanField(default=False, verbose_name='Initialized'), + ), + migrations.AlterField( + model_name='appletpublication', + name='applet', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='publications', to='terminal.applet', verbose_name='Applet'), + ), + migrations.AlterField( + model_name='appletpublication', + name='host', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='publications', to='terminal.applethost', verbose_name='Host'), + ), + migrations.DeleteModel( + name='AppletHostDeployment', + ), + ] diff --git a/apps/terminal/models/applet/applet.py b/apps/terminal/models/applet/applet.py index 0969204b7..9dde129e7 100644 --- a/apps/terminal/models/applet/applet.py +++ b/apps/terminal/models/applet/applet.py @@ -26,6 +26,7 @@ class Applet(JMSBaseModel): protocols = models.JSONField(default=list, verbose_name=_('Protocol')) tags = models.JSONField(default=list, verbose_name=_('Tags')) comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) + hosts = models.ManyToManyField(through_fields=('applet', 'host'), through='AppletPublication', to='AppletHost', verbose_name=_('Hosts')) def __str__(self): return self.name @@ -51,8 +52,8 @@ class Applet(JMSBaseModel): class AppletPublication(JMSBaseModel): - applet = models.ForeignKey('Applet', on_delete=models.PROTECT, verbose_name=_('Applet')) - host = models.ForeignKey('AppletHost', on_delete=models.PROTECT, verbose_name=_('Host')) + applet = models.ForeignKey('Applet', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Applet')) + host = models.ForeignKey('AppletHost', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Host')) status = models.CharField(max_length=16, verbose_name=_('Status')) comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) diff --git a/apps/terminal/models/applet/host.py b/apps/terminal/models/applet/host.py index 2fee8d15d..7733e7b0f 100644 --- a/apps/terminal/models/applet/host.py +++ b/apps/terminal/models/applet/host.py @@ -1,15 +1,17 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from common.db.models import JMSBaseModel from assets.models import Host +from ops.ansible import PlaybookRunner, JMSInventory -__all__ = ['AppletHost', 'AppletHostDeployment'] +__all__ = ['AppletHost'] class AppletHost(Host): account_automation = models.BooleanField(default=False, verbose_name=_('Account automation')) + initialized = models.BooleanField(default=False, verbose_name=_('Initialized')) + date_inited = models.DateTimeField(null=True, blank=True, verbose_name=_('Date initialized')) date_synced = models.DateTimeField(null=True, blank=True, verbose_name=_('Date synced')) status = models.CharField(max_length=16, verbose_name=_('Status')) applets = models.ManyToManyField( @@ -17,17 +19,11 @@ class AppletHost(Host): through='AppletPublication', through_fields=('host', 'applet'), ) + def deploy(self): + inventory = JMSInventory([self]) + playbook = PlaybookRunner(inventory, 'applets.yml') + playbook.run() + def __str__(self): return self.name - -class AppletHostDeployment(JMSBaseModel): - host = models.ForeignKey('AppletHost', on_delete=models.CASCADE, verbose_name=_('Hosting')) - status = models.CharField(max_length=16, verbose_name=_('Status')) - comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) - - def __str__(self): - return self.host - - def start(self): - pass diff --git a/apps/terminal/serializers/applet.py b/apps/terminal/serializers/applet.py index 97f8fae64..802b0a304 100644 --- a/apps/terminal/serializers/applet.py +++ b/apps/terminal/serializers/applet.py @@ -5,12 +5,12 @@ from common.drf.fields import ObjectRelatedField, LabeledChoiceField from common.validators import ProjectUniqueValidator from assets.models import Platform from assets.serializers import HostSerializer -from ..models import Applet, AppletPublication, AppletHost, AppletHostDeployment +from ..models import Applet, AppletPublication, AppletHost __all__ = [ 'AppletSerializer', 'AppletPublicationSerializer', - 'AppletHostSerializer', 'AppletHostDeploymentSerializer', + 'AppletHostSerializer', 'AppletUploadSerializer' ] @@ -85,14 +85,3 @@ class AppletHostSerializer(HostSerializer): validators.append(uniq_validator) return validators - -class AppletHostDeploymentSerializer(serializers.ModelSerializer): - host = ObjectRelatedField(queryset=AppletHost.objects.all()) - - class Meta: - model = AppletHostDeployment - fields_mini = ['id', 'host'] - read_only_fields = ['date_created', 'date_updated'] - fields = fields_mini + [ - 'status', 'comment', - ] + read_only_fields diff --git a/apps/terminal/tasks.py b/apps/terminal/tasks.py index 59cf00fa4..e05d673e5 100644 --- a/apps/terminal/tasks.py +++ b/apps/terminal/tasks.py @@ -14,7 +14,7 @@ from common.utils import get_log_keep_day from ops.celery.decorator import ( register_as_period_task, after_app_ready_start, after_app_shutdown_clean_periodic ) -from .models import Status, Session, Command, Task +from .models import Status, Session, Command, Task, AppletHost from .backends import server_replay_storage from .utils import find_session_replay_local @@ -99,3 +99,9 @@ def upload_session_replay_to_external_storage(session_id): except: pass return + + +@shared_task +def run_applet_host_deployment(did): + host = AppletHost.objects.get(id=did) + host.deploy() diff --git a/apps/terminal/urls/api_urls.py b/apps/terminal/urls/api_urls.py index 8bed4f604..1d369cc52 100644 --- a/apps/terminal/urls/api_urls.py +++ b/apps/terminal/urls/api_urls.py @@ -26,8 +26,7 @@ router.register(r'endpoints', api.EndpointViewSet, 'endpoint') router.register(r'endpoint-rules', api.EndpointRuleViewSet, 'endpoint-rule') router.register(r'applets', api.AppletViewSet, 'applet') router.register(r'applet-hosts', api.AppletHostViewSet, 'applet-host') -router.register(r'applet-publication', api.AppletPublicationViewSet, 'applet-publication') -router.register(r'applet-host-deployment', api.AppletHostDeploymentViewSet, 'applet-host-deployment') +router.register(r'applet-publications', api.AppletPublicationViewSet, 'applet-publication') urlpatterns = [ @@ -46,10 +45,6 @@ urlpatterns = [ path('command-storages//test-connective/', api.CommandStorageTestConnectiveApi.as_view(), name='command-storage-test-connective'), # components path('components/metrics/', api.ComponentsMetricsAPIView.as_view(), name='components-metrics'), - # v2: get session's replay - # path('v2/sessions//replay/', - # api.SessionReplayV2ViewSet.as_view({'get': 'retrieve'}), - # name='session-replay-v2'), ] old_version_urlpatterns = [