From cf81f08b7ab8542eeac9fbae6cc3b4d906f61b31 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 1 Nov 2022 11:52:51 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E9=83=A8=E7=BD=B2=20?= =?UTF-8?q?host?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/automations/base/manager.py | 8 ++-- .../automations/change_secret/manager.py | 2 +- apps/assets/models/asset/common.py | 2 +- apps/assets/models/asset/database.py | 2 +- apps/assets/models/automations/base.py | 2 +- apps/assets/serializers/asset/common.py | 4 +- apps/common/const/choices.py | 9 +++++ apps/ops/ansible/callback.py | 13 ++++++- apps/ops/ansible/inventory.py | 38 ++++++++++--------- apps/ops/models/adhoc.py | 4 +- apps/ops/models/base.py | 10 ++--- apps/ops/models/celery.py | 1 + apps/ops/models/playbook.py | 4 +- apps/orgs/mixins/models.py | 23 +++++------ apps/orgs/utils.py | 26 ++++++------- apps/terminal/api/applet/host.py | 32 ++++++++-------- .../{manager.py => __init__.py} | 29 ++++++++++++-- .../migrations/0054_auto_20221027_1125.py | 2 +- ...028_1544.py => 0055_auto_20221031_1848.py} | 21 ++++++---- apps/terminal/models/applet/applet.py | 1 + apps/terminal/models/applet/host.py | 27 ++++++++----- apps/terminal/serializers/applet.py | 16 ++++++-- apps/terminal/tasks.py | 17 +++++++-- apps/terminal/urls/api_urls.py | 1 + 24 files changed, 186 insertions(+), 108 deletions(-) rename apps/terminal/automations/deploy_applet_host/{manager.py => __init__.py} (63%) rename apps/terminal/migrations/{0055_auto_20221028_1544.py => 0055_auto_20221031_1848.py} (71%) diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index a927987d1..758dea52c 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -8,7 +8,6 @@ from collections import defaultdict from django.conf import settings from django.utils import timezone -from django.db.models import Model from django.utils.translation import gettext as _ from common.utils import get_logger @@ -115,9 +114,9 @@ class BasePlaybookManager: method_attr = '{}_method'.format(self.__class__.method_type()) method_enabled = automation and \ - getattr(automation, enabled_attr) and \ - getattr(automation, method_attr) and \ - getattr(automation, method_attr) in self.method_id_meta_mapper + getattr(automation, enabled_attr) and \ + getattr(automation, method_attr) and \ + getattr(automation, method_attr) in self.method_id_meta_mapper if not method_enabled: host['error'] = _('{} disabled'.format(self.__class__.method_type())) @@ -132,6 +131,7 @@ class BasePlaybookManager: def generate_private_key_path(secret, path_dir): key_name = '.' + md5(secret.encode('utf-8')).hexdigest() key_path = os.path.join(path_dir, key_name) + if not os.path.exists(key_path): ssh_key_string_to_obj(secret, password=None).write_private_key_file(key_path) os.chmod(key_path, 0o400) diff --git a/apps/assets/automations/change_secret/manager.py b/apps/assets/automations/change_secret/manager.py index b8868acd1..4f77dfe85 100644 --- a/apps/assets/automations/change_secret/manager.py +++ b/apps/assets/automations/change_secret/manager.py @@ -154,7 +154,7 @@ class ChangeSecretManager(BasePlaybookManager): recorder = self.name_recorder_mapper.get(host) if not recorder: return - recorder.status = 'succeed' + recorder.status = 'success' recorder.date_finished = timezone.now() recorder.save() diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index fd29bf6a4..44fbddc23 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -106,7 +106,7 @@ class Asset(NodesRelationMixin, AbsConnectivity, JMSOrgBaseModel): return '{0.name}({0.address})'.format(self) @property - def category_property(self): + def specific(self): if not hasattr(self, self.category): return {} instance = getattr(self, self.category) diff --git a/apps/assets/models/asset/database.py b/apps/assets/models/asset/database.py index de6b6a758..6aef15a8f 100644 --- a/apps/assets/models/asset/database.py +++ b/apps/assets/models/asset/database.py @@ -15,7 +15,7 @@ class Database(Asset): return self.address @property - def category_property(self): + def specific(self): return { 'db_name': self.db_name, } diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index aabfd241f..a60a0e060 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -3,7 +3,7 @@ from celery import current_task from django.db import models from django.utils.translation import ugettext_lazy as _ -from common.const.choices import Trigger +from common.const.choices import Trigger, Status from common.mixins.models import CommonModelMixin from common.db.fields import EncryptJsonDictTextField from orgs.mixins.models import OrgModelMixin diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index b6204e843..d05b430cb 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -77,7 +77,7 @@ class AssetSerializer(OrgResourceSerializerMixin, WritableNestedModelSerializer) 'nodes', 'labels', 'accounts', 'protocols', 'nodes_display', ] read_only_fields = [ - 'category', 'type', 'category_property', + 'category', 'type', 'specific', 'connectivity', 'date_verified', 'created_by', 'date_created', ] @@ -90,7 +90,7 @@ class AssetSerializer(OrgResourceSerializerMixin, WritableNestedModelSerializer) def get_field_names(self, declared_fields, info): names = super().get_field_names(declared_fields, info) if self.__class__.__name__ != 'AssetSerializer': - names.remove('category_property') + names.remove('specific') return names @classmethod diff --git a/apps/common/const/choices.py b/apps/common/const/choices.py index f752bd50e..fba8418df 100644 --- a/apps/common/const/choices.py +++ b/apps/common/const/choices.py @@ -9,3 +9,12 @@ AUDITOR = 'Auditor' class Trigger(models.TextChoices): manual = 'manual', _('Manual trigger') timing = 'timing', _('Timing trigger') + + +class Status(models.TextChoices): + pending = 'pending', _("Pending") + running = 'running', _("Running") + success = 'success', _("Success") + failed = 'failed', _("Failed") + error = 'error', _("Error") + canceled = 'canceled', _("Canceled") diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index c613d5080..3f794a194 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -2,6 +2,14 @@ from collections import defaultdict class DefaultCallback: + STATUS_MAPPER = { + 'successful': 'success', + 'failure': 'failed', + 'running': 'running', + 'pending': 'pending', + 'unknown': 'unknown' + } + def __init__(self): self.result = dict( ok=defaultdict(dict), @@ -27,7 +35,7 @@ class DefaultCallback: return results def is_success(self): - return self.status != 'successful' + return self.status != 'success' def event_handler(self, data, **kwargs): event = data.get('event', None) @@ -131,4 +139,5 @@ class DefaultCallback: pass def status_handler(self, data, **kwargs): - self.status = data.get('status', 'unknown') + status = data.get('status', '') + self.status = self.STATUS_MAPPER.get(status, 'unknown') diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 8a1ae0634..1e006ae77 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -9,13 +9,12 @@ __all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account_policy='smart', - account_prefer='root,administrator', - host_callback=None): + def __init__(self, assets, account_policy='privileged_first', + account_prefer='root,Administrator', host_callback=None): """ :param assets: :param account_prefer: account username name if not set use account_policy - :param account_policy: smart, privileged_must, privileged_first + :param account_policy: privileged_only, privileged_first, skip """ self.assets = self.clean_assets(assets) self.account_prefer = account_prefer @@ -105,7 +104,7 @@ class JMSInventory: 'id': str(asset.id), 'name': asset.name, 'address': asset.address, 'type': asset.type, 'category': asset.category, 'protocol': asset.protocol, 'port': asset.port, - 'category_property': asset.category_property, + 'specific': asset.specific, 'protocols': [{'name': p.name, 'port': p.port} for p in protocols], }, 'jms_account': { @@ -137,24 +136,27 @@ class JMSInventory: def select_account(self, asset): accounts = list(asset.accounts.all()) account_selected = None - account_username = self.account_prefer + account_usernames = self.account_prefer if isinstance(self.account_prefer, str): - account_username = self.account_prefer.split(',') + account_usernames = self.account_prefer.split(',') - if account_username: - for username in account_username: - account_matched = list(filter(lambda account: account.username == username, accounts)) - if account_matched: - account_selected = account_matched[0] - break + # 优先使用提供的名称 + if account_usernames: + account_matched = list(filter(lambda account: account.username in account_usernames, accounts)) + account_selected = account_matched[0] if account_matched else None - if not account_selected: - if self.account_policy in ['privileged_must', 'privileged_first']: - account_matched = list(filter(lambda account: account.privileged, accounts)) - account_selected = account_matched[0] if account_matched else None + if account_selected or self.account_policy == 'skip': + return account_selected - if not account_selected and self.account_policy == 'privileged_first': + if self.account_policy in ['privileged_only', 'privileged_first']: + account_matched = list(filter(lambda account: account.privileged, accounts)) + account_selected = account_matched[0] if account_matched else None + + if account_selected: + return account_selected + + if self.account_policy == 'privileged_first': account_selected = accounts[0] if accounts else None return account_selected diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 2273a21b6..22fd0f054 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -5,7 +5,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from common.utils import get_logger -from .base import BaseAnsibleTask, BaseAnsibleExecution +from .base import BaseAnsibleJob, BaseAnsibleExecution from ..ansible import AdHocRunner __all__ = ["AdHoc", "AdHocExecution"] @@ -14,7 +14,7 @@ __all__ = ["AdHoc", "AdHocExecution"] logger = get_logger(__file__) -class AdHoc(BaseAnsibleTask): +class AdHoc(BaseAnsibleJob): pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') module = models.CharField(max_length=128, default='shell', verbose_name=_('Module')) args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py index 4ff69277f..a37982673 100644 --- a/apps/ops/models/base.py +++ b/apps/ops/models/base.py @@ -12,7 +12,7 @@ from ..ansible.inventory import JMSInventory from ..mixin import PeriodTaskModelMixin -class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): +class BaseAnsibleJob(PeriodTaskModelMixin, JMSOrgBaseModel): owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) @@ -46,7 +46,7 @@ class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): class BaseAnsibleExecution(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4) status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') - task = models.ForeignKey(BaseAnsibleTask, on_delete=models.CASCADE, related_name='executions', null=True) + task = models.ForeignKey(BaseAnsibleJob, on_delete=models.CASCADE, related_name='executions', null=True) result = models.JSONField(blank=True, null=True, verbose_name=_('Result')) summary = models.JSONField(default=dict, verbose_name=_('Summary')) creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) @@ -86,7 +86,7 @@ class BaseAnsibleExecution(models.Model): def set_result(self, cb): status_mapper = { - 'successful': 'succeeded', + 'successful': 'success', } this = self.__class__.objects.get(id=self.id) this.status = status_mapper.get(cb.status, cb.status) @@ -112,11 +112,11 @@ class BaseAnsibleExecution(models.Model): @property def is_finished(self): - return self.status in ['succeeded', 'failed'] + return self.status in ['success', 'failed'] @property def is_success(self): - return self.status == 'succeeded' + return self.status == 'success' @property def time_cost(self): diff --git a/apps/ops/models/celery.py b/apps/ops/models/celery.py index 2cc989fc3..55f05129f 100644 --- a/apps/ops/models/celery.py +++ b/apps/ops/models/celery.py @@ -23,6 +23,7 @@ class CeleryTask(models.Model): "comment": getattr(task, 'comment', None), "queue": getattr(task, 'queue', 'default') } + @property def state(self): last_five_executions = CeleryTaskExecution.objects.filter(name=self.name).order_by('-date_published')[:5] diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py index 0701ed13e..a0c11db3b 100644 --- a/apps/ops/models/playbook.py +++ b/apps/ops/models/playbook.py @@ -2,7 +2,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from orgs.mixins.models import JMSOrgBaseModel -from .base import BaseAnsibleExecution, BaseAnsibleTask +from .base import BaseAnsibleExecution, BaseAnsibleJob class PlaybookTemplate(JMSOrgBaseModel): @@ -19,7 +19,7 @@ class PlaybookTemplate(JMSOrgBaseModel): unique_together = [('org_id', 'name')] -class Playbook(BaseAnsibleTask): +class Playbook(BaseAnsibleJob): path = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) owner = models.ForeignKey('users.User', verbose_name=_("Owner"), on_delete=models.SET_NULL, null=True) comment = models.TextField(blank=True, verbose_name=_("Comment")) diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index 63b78bd23..80674ddc6 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -20,16 +20,15 @@ __all__ = [ class OrgManager(models.Manager): - def all_group_by_org(self): from ..models import Organization orgs = list(Organization.objects.all()) - querysets = {} + org_queryset = {} for org in orgs: org_id = org.id queryset = super(OrgManager, self).get_queryset().filter(org_id=org_id) - querysets[org] = queryset - return querysets + org_queryset[org] = queryset + return org_queryset def get_queryset(self): queryset = super(OrgManager, self).get_queryset() @@ -46,7 +45,7 @@ class OrgManager(models.Manager): for obj in objs: if org.is_root(): if not obj.org_id: - raise ValidationError('Please save in a organization') + raise ValidationError('Please save in a org') else: obj.org_id = org.id return super().bulk_create(objs, batch_size, ignore_conflicts) @@ -54,20 +53,24 @@ class OrgManager(models.Manager): class OrgModelMixin(models.Model): org_id = models.CharField( - max_length=36, blank=True, default='', verbose_name=_("Organization"), db_index=True + max_length=36, blank=True, default='', + verbose_name=_("Organization"), db_index=True ) objects = OrgManager() - sep = '@' def save(self, *args, **kwargs): - org = get_current_org() + locking_org = getattr(self, 'locking_org', None) + if locking_org: + org = Organization.get_instance(locking_org) + else: + org = get_current_org() # 这里不可以优化成, 因为 root 组织下可以设置组织 id 来保存 # if org.is_root() and not self.org_id: # raise ... if org.is_root(): if not self.org_id: - raise ValidationError('Please save in a organization') + raise ValidationError('Please save in a org') else: self.org_id = org.id return super().save(*args, **kwargs) @@ -87,8 +90,6 @@ class OrgModelMixin(models.Model): name = getattr(self, attr) elif hasattr(self, 'name'): name = self.name - elif hasattr(self, 'name'): - name = self.hostname return name + self.sep + self.org_name def validate_unique(self, exclude=None): diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index 0ea4085e7..a80c7854d 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -103,22 +103,20 @@ def tmp_to_builtin_org(system=0, default=0): set_current_org(ori_org) -def get_org_filters(): - kwargs = {} - - _current_org = get_current_org() - if _current_org is None: - return kwargs - if _current_org.is_root(): - return kwargs - kwargs['org_id'] = _current_org.id - return kwargs - - def filter_org_queryset(queryset): - kwargs = get_org_filters() + locking_org = getattr(queryset.model, 'LOCKING_ORG', None) + if locking_org: + org = Organization.get_instance(locking_org) + else: + org = get_current_org() + + if org is None: + kwargs = {} + elif org.is_root(): + kwargs = {} + else: + kwargs = {'org_id': org.id} - # # lines = traceback.format_stack() # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>") # for line in lines[-10:-1]: diff --git a/apps/terminal/api/applet/host.py b/apps/terminal/api/applet/host.py index ea8831b8f..d9c7cde7e 100644 --- a/apps/terminal/api/applet/host.py +++ b/apps/terminal/api/applet/host.py @@ -2,29 +2,17 @@ from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response -from orgs.utils import tmp_to_builtin_org from terminal import serializers -from terminal.models import AppletHost, Applet +from terminal.models import AppletHost, Applet, AppletHostDeployment from terminal.tasks import run_applet_host_deployment -__all__ = ['AppletHostViewSet'] + +__all__ = ['AppletHostViewSet', 'AppletHostDeploymentViewSet'] class AppletHostViewSet(viewsets.ModelViewSet): serializer_class = serializers.AppletHostSerializer - - def get_queryset(self): - return AppletHost.objects.all() - - def dispatch(self, request, *args, **kwargs): - with tmp_to_builtin_org(system=1): - return super().dispatch(request, *args, **kwargs) - - @action(methods=['post'], detail=True) - def deploy(self, request): - from terminal.automations.deploy_applet_host.manager import DeployAppletHostManager - manager = DeployAppletHostManager(self) - manager.run() + queryset = AppletHost.objects.all() @action(methods=['get'], detail=True, url_path='') def not_published_applets(self, request, *args, **kwargs): @@ -33,3 +21,15 @@ class AppletHostViewSet(viewsets.ModelViewSet): serializer = serializers.AppletSerializer(applets, many=True) return Response(serializer.data) + +class AppletHostDeploymentViewSet(viewsets.ModelViewSet): + serializer_class = serializers.AppletHostDeploymentSerializer + queryset = AppletHostDeployment.objects.all() + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + instance = serializer.save() + task = run_applet_host_deployment.delay(instance.id) + return Response({'task': str(task.id)}, status=201) + diff --git a/apps/terminal/automations/deploy_applet_host/manager.py b/apps/terminal/automations/deploy_applet_host/__init__.py similarity index 63% rename from apps/terminal/automations/deploy_applet_host/manager.py rename to apps/terminal/automations/deploy_applet_host/__init__.py index 22a8510af..287d44b3e 100644 --- a/apps/terminal/automations/deploy_applet_host/manager.py +++ b/apps/terminal/automations/deploy_applet_host/__init__.py @@ -1,16 +1,21 @@ import os import datetime import shutil + +from django.utils import timezone from django.conf import settings +from common.utils import get_logger +from common.db.utils import safe_db_connection from ops.ansible import PlaybookRunner, JMSInventory +logger = get_logger(__name__) CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) class DeployAppletHostManager: - def __init__(self, applet_host): - self.applet_host = applet_host + def __init__(self, deployment): + self.deployment = deployment self.run_dir = self.get_run_dir() @staticmethod @@ -28,16 +33,32 @@ class DeployAppletHostManager: return playbook_dst def generate_inventory(self): - inventory = JMSInventory([self.applet_host], account_policy='privileged_only') + inventory = JMSInventory([self.deployment.host], account_policy='privileged_only') inventory_dir = os.path.join(self.run_dir, 'inventory') inventory_path = os.path.join(inventory_dir, 'hosts.yml') inventory.write_to_file(inventory_path) return inventory_path - def run(self, **kwargs): + def _run(self, **kwargs): inventory = self.generate_inventory() playbook = self.generate_playbook() runner = PlaybookRunner( inventory=inventory, playbook=playbook, project_dir=self.run_dir ) return runner.run(**kwargs) + + def run(self, **kwargs): + try: + self.deployment.date_start = timezone.now() + cb = self._run(**kwargs) + self.deployment.status = cb.status + except Exception as e: + logger.error("Error: {}".format(e)) + self.deployment.status = 'error' + finally: + self.deployment.date_finished = timezone.now() + with safe_db_connection(): + self.deployment.save() + + + diff --git a/apps/terminal/migrations/0054_auto_20221027_1125.py b/apps/terminal/migrations/0054_auto_20221027_1125.py index 418cbe993..4589d6970 100644 --- a/apps/terminal/migrations/0054_auto_20221027_1125.py +++ b/apps/terminal/migrations/0054_auto_20221027_1125.py @@ -73,7 +73,7 @@ class Migration(migrations.Migration): ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('status', models.CharField(max_length=16, verbose_name='Status')), + ('status', models.CharField(max_length=16, default='', verbose_name='Status')), ('comment', models.TextField(blank=True, default='', verbose_name='Comment')), ('host', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='terminal.applethost', verbose_name='Hosting')), ], diff --git a/apps/terminal/migrations/0055_auto_20221028_1544.py b/apps/terminal/migrations/0055_auto_20221031_1848.py similarity index 71% rename from apps/terminal/migrations/0055_auto_20221028_1544.py rename to apps/terminal/migrations/0055_auto_20221031_1848.py index 3aeeff3f6..e36ed5a9b 100644 --- a/apps/terminal/migrations/0055_auto_20221028_1544.py +++ b/apps/terminal/migrations/0055_auto_20221031_1848.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.14 on 2022-10-28 07:44 +# Generated by Django 3.2.14 on 2022-10-31 10:48 from django.db import migrations, models import django.db.models.deletion @@ -19,12 +19,22 @@ class Migration(migrations.Migration): migrations.AddField( model_name='applethost', name='date_inited', - field=models.DateTimeField(blank=True, null=True, verbose_name='Date initialized'), + field=models.DateTimeField(blank=True, null=True, verbose_name='Date inited'), ), migrations.AddField( model_name='applethost', - name='initialized', - field=models.BooleanField(default=False, verbose_name='Initialized'), + name='inited', + field=models.BooleanField(default=False, verbose_name='Inited'), + ), + migrations.AddField( + model_name='applethostdeployment', + name='date_finished', + field=models.DateTimeField(null=True, verbose_name='Date finished'), + ), + migrations.AddField( + model_name='applethostdeployment', + name='date_start', + field=models.DateTimeField(db_index=True, null=True, verbose_name='Date start'), ), migrations.AlterField( model_name='appletpublication', @@ -36,7 +46,4 @@ class Migration(migrations.Migration): name='host', field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='publications', to='terminal.applethost', verbose_name='Host'), ), - migrations.DeleteModel( - name='AppletHostDeployment', - ), ] diff --git a/apps/terminal/models/applet/applet.py b/apps/terminal/models/applet/applet.py index 9dde129e7..14f363517 100644 --- a/apps/terminal/models/applet/applet.py +++ b/apps/terminal/models/applet/applet.py @@ -55,6 +55,7 @@ class AppletPublication(JMSBaseModel): applet = models.ForeignKey('Applet', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Applet')) host = models.ForeignKey('AppletHost', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Host')) status = models.CharField(max_length=16, verbose_name=_('Status')) + published = models.BooleanField(default=False, verbose_name=_('Published')) comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) class Meta: diff --git a/apps/terminal/models/applet/host.py b/apps/terminal/models/applet/host.py index 7733e7b0f..e1edc2239 100644 --- a/apps/terminal/models/applet/host.py +++ b/apps/terminal/models/applet/host.py @@ -1,17 +1,19 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +from common.db.models import JMSBaseModel from assets.models import Host -from ops.ansible import PlaybookRunner, JMSInventory -__all__ = ['AppletHost'] +__all__ = ['AppletHost', 'AppletHostDeployment'] class AppletHost(Host): + LOCKING_ORG = 'SYSTEM' + account_automation = models.BooleanField(default=False, verbose_name=_('Account automation')) - initialized = models.BooleanField(default=False, verbose_name=_('Initialized')) - date_inited = models.DateTimeField(null=True, blank=True, verbose_name=_('Date initialized')) + inited = models.BooleanField(default=False, verbose_name=_('Inited')) + date_inited = models.DateTimeField(null=True, blank=True, verbose_name=_('Date inited')) date_synced = models.DateTimeField(null=True, blank=True, verbose_name=_('Date synced')) status = models.CharField(max_length=16, verbose_name=_('Status')) applets = models.ManyToManyField( @@ -19,11 +21,18 @@ class AppletHost(Host): through='AppletPublication', through_fields=('host', 'applet'), ) - def deploy(self): - inventory = JMSInventory([self]) - playbook = PlaybookRunner(inventory, 'applets.yml') - playbook.run() - def __str__(self): return self.name + +class AppletHostDeployment(JMSBaseModel): + host = models.ForeignKey('AppletHost', on_delete=models.CASCADE, verbose_name=_('Hosting')) + status = models.CharField(max_length=16, default='', verbose_name=_('Status')) + date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) + date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished")) + comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) + + def start(self, **kwargs): + from ...automations.deploy_applet_host import DeployAppletHostManager + manager = DeployAppletHostManager(self) + manager.run(**kwargs) diff --git a/apps/terminal/serializers/applet.py b/apps/terminal/serializers/applet.py index 802b0a304..aad7e10ae 100644 --- a/apps/terminal/serializers/applet.py +++ b/apps/terminal/serializers/applet.py @@ -5,13 +5,13 @@ from common.drf.fields import ObjectRelatedField, LabeledChoiceField from common.validators import ProjectUniqueValidator from assets.models import Platform from assets.serializers import HostSerializer -from ..models import Applet, AppletPublication, AppletHost +from ..models import Applet, AppletPublication, AppletHost, AppletHostDeployment __all__ = [ 'AppletSerializer', 'AppletPublicationSerializer', - 'AppletHostSerializer', - 'AppletUploadSerializer' + 'AppletHostSerializer', 'AppletHostDeploymentSerializer', + 'AppletUploadSerializer', ] @@ -85,3 +85,13 @@ class AppletHostSerializer(HostSerializer): validators.append(uniq_validator) return validators + +class AppletHostDeploymentSerializer(serializers.ModelSerializer): + class Meta: + model = AppletHostDeployment + fields_mini = ['id', 'host', 'status'] + read_only_fields = [ + 'status', 'date_created', 'date_updated', + 'date_start', 'date_finished' + ] + fields = fields_mini + ['comment'] + read_only_fields diff --git a/apps/terminal/tasks.py b/apps/terminal/tasks.py index e05d673e5..396369d16 100644 --- a/apps/terminal/tasks.py +++ b/apps/terminal/tasks.py @@ -12,9 +12,14 @@ from django.core.files.storage import default_storage from common.utils import get_log_keep_day from ops.celery.decorator import ( - register_as_period_task, after_app_ready_start, after_app_shutdown_clean_periodic + register_as_period_task, after_app_ready_start, + after_app_shutdown_clean_periodic ) -from .models import Status, Session, Command, Task, AppletHost +from .models import ( + Status, Session, Command, Task, AppletHost, + AppletHostDeployment +) +from orgs.utils import tmp_to_builtin_org from .backends import server_replay_storage from .utils import find_session_replay_local @@ -84,16 +89,19 @@ def upload_session_replay_to_external_storage(session_id): if not session: logger.error(f'Session db item not found: {session_id}') return + local_path, foobar = find_session_replay_local(session) if not local_path: logger.error(f'Session replay not found, may be upload error: {local_path}') return + abs_path = default_storage.path(local_path) remote_path = session.get_relative_path_by_local_path(abs_path) ok, err = server_replay_storage.upload(abs_path, remote_path) if not ok: logger.error(f'Session replay upload to external error: {err}') return + try: default_storage.delete(local_path) except: @@ -103,5 +111,6 @@ def upload_session_replay_to_external_storage(session_id): @shared_task def run_applet_host_deployment(did): - host = AppletHost.objects.get(id=did) - host.deploy() + with tmp_to_builtin_org(system=1): + deployment = AppletHostDeployment.objects.get(id=did) + deployment.start() diff --git a/apps/terminal/urls/api_urls.py b/apps/terminal/urls/api_urls.py index 1d369cc52..9a573acc5 100644 --- a/apps/terminal/urls/api_urls.py +++ b/apps/terminal/urls/api_urls.py @@ -27,6 +27,7 @@ router.register(r'endpoint-rules', api.EndpointRuleViewSet, 'endpoint-rule') router.register(r'applets', api.AppletViewSet, 'applet') router.register(r'applet-hosts', api.AppletHostViewSet, 'applet-host') router.register(r'applet-publications', api.AppletPublicationViewSet, 'applet-publication') +router.register(r'applet-host-deployments', api.AppletHostDeploymentViewSet, 'applet-host-deployment') urlpatterns = [