diff --git a/apps/authentication/migrations/0013_auto_20221025_1908.py b/apps/authentication/migrations/0013_auto_20221025_1908.py deleted file mode 100644 index 452063f35..000000000 --- a/apps/authentication/migrations/0013_auto_20221025_1908.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.14 on 2022-10-25 11:08 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('assets', '0110_auto_20221021_1506'), - ('authentication', '0012_auto_20220816_1629'), - ] - - operations = [ - migrations.RemoveField( - model_name='connectiontoken', - name='type', - ), - migrations.AddField( - model_name='connectiontoken', - name='account_display', - field=models.CharField(default='', max_length=128, verbose_name='Account display'), - ), - migrations.AlterField( - model_name='connectiontoken', - name='account', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='connection_tokens', to='assets.account', verbose_name='Account'), - ), - ] diff --git a/apps/authentication/migrations/0013_remove_connectiontoken_type.py b/apps/authentication/migrations/0013_remove_connectiontoken_type.py new file mode 100644 index 000000000..52c6813dc --- /dev/null +++ b/apps/authentication/migrations/0013_remove_connectiontoken_type.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.14 on 2022-10-26 08:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('authentication', '0012_auto_20220816_1629'), + ] + + operations = [ + migrations.RemoveField( + model_name='connectiontoken', + name='type', + ), + ] diff --git a/apps/authentication/models.py b/apps/authentication/models.py index 642443e02..765f38ef9 100644 --- a/apps/authentication/models.py +++ b/apps/authentication/models.py @@ -78,11 +78,7 @@ class ConnectionToken(OrgModelMixin, JMSBaseModel): related_name='connection_tokens', null=True, blank=True ) asset_display = models.CharField(max_length=128, default='', verbose_name=_("Asset display")) - account = models.ForeignKey( - 'assets.Account', on_delete=models.SET_NULL, verbose_name=_('Account'), - related_name='connection_tokens', null=True, blank=True - ) - account_display = models.CharField(max_length=128, default='', verbose_name=_("Account display")) + account = models.CharField(max_length=128, default='', verbose_name=_("Account")) class Meta: ordering = ('-date_expired',) @@ -127,7 +123,6 @@ class ConnectionToken(OrgModelMixin, JMSBaseModel): def check_valid(self): from perms.utils.permission import validate_permission as asset_validate_permission - from perms.utils.application.permission import validate_permission as app_validate_permission if self.is_expired: is_valid = False @@ -143,45 +138,30 @@ class ConnectionToken(OrgModelMixin, JMSBaseModel): error = _('User invalid, disabled or expired') return is_valid, error - if not self.system_user: + if not self.account: is_valid = False - error = _('System user not exists') + error = _('Account not exists') return is_valid, error - if self.is_type(self.Type.asset): - if not self.asset: - is_valid = False - error = _('Asset not exists') - return is_valid, error - if not self.asset.is_active: - is_valid = False - error = _('Asset inactive') - return is_valid, error - has_perm, actions, expired_at = asset_validate_permission( - self.user, self.asset, self.system_user - ) - if not has_perm: - is_valid = False - error = _('User has no permission to access asset or permission expired') - return is_valid, error - self.actions = actions - self.expired_at = expired_at + if not self.asset: + is_valid = False + error = _('Asset not exists') + return is_valid, error - elif self.is_type(self.Type.application): - if not self.application: - is_valid = False - error = _('Application not exists') - return is_valid, error - has_perm, actions, expired_at = app_validate_permission( - self.user, self.application, self.system_user - ) - if not has_perm: - is_valid = False - error = _('User has no permission to access application or permission expired') - return is_valid, error - self.actions = actions - self.expired_at = expired_at + if not self.asset.is_active: + is_valid = False + error = _('Asset inactive') + return is_valid, error + has_perm, actions, expired_at = asset_validate_permission( + self.user, self.asset, self.account + ) + if not has_perm: + is_valid = False + error = _('User has no permission to access asset or permission expired') + return is_valid, error + self.actions = actions + self.expired_at = expired_at return True, '' @lazyproperty diff --git a/apps/common/drf/serializers/common.py b/apps/common/drf/serializers/common.py index 688e1bfcc..a522e3a82 100644 --- a/apps/common/drf/serializers/common.py +++ b/apps/common/drf/serializers/common.py @@ -12,7 +12,7 @@ from .mixin import BulkListSerializerMixin, BulkSerializerMixin __all__ = [ 'MethodSerializer', 'EmptySerializer', 'BulkModelSerializer', - 'AdaptedBulkListSerializer', 'CeleryTaskSerializer', + 'AdaptedBulkListSerializer', 'CeleryTaskExecutionSerializer', 'WritableNestedModelSerializer', 'GroupedChoiceSerializer', ] @@ -73,7 +73,7 @@ class AdaptedBulkListSerializer(BulkListSerializerMixin, BulkListSerializer): pass -class CeleryTaskSerializer(serializers.Serializer): +class CeleryTaskExecutionSerializer(serializers.Serializer): task = serializers.CharField(read_only=True) diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 8644ac5d2..71e818fbb 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -5,7 +5,7 @@ from django.shortcuts import get_object_or_404 from rest_framework import viewsets, generics from rest_framework.views import Response -from common.drf.serializers import CeleryTaskSerializer +from common.drf.serializers import CeleryTaskExecutionSerializer from ..models import AdHoc, AdHocExecution from ..serializers import ( AdHocSerializer, diff --git a/apps/ops/api/celery.py b/apps/ops/api/celery.py index cd452c471..61d13db17 100644 --- a/apps/ops/api/celery.py +++ b/apps/ops/api/celery.py @@ -4,6 +4,7 @@ import os import re +from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ from rest_framework import viewsets from celery.result import AsyncResult @@ -12,20 +13,21 @@ from django_celery_beat.models import PeriodicTask from common.permissions import IsValidUser from common.api import LogTailApi -from ..models import CeleryTask +from ..models import CeleryTaskExecution, CeleryTask from ..serializers import CeleryResultSerializer, CeleryPeriodTaskSerializer from ..celery.utils import get_celery_task_log_path from ..ansible.utils import get_ansible_task_log_path from common.mixins.api import CommonApiMixin - __all__ = [ - 'CeleryTaskLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet', - 'AnsibleTaskLogApi', + 'CeleryTaskExecutionLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet', + 'AnsibleTaskLogApi', 'CeleryTaskViewSet', 'CeleryTaskExecutionViewSet' ] +from ..serializers.celery import CeleryTaskSerializer, CeleryTaskExecutionSerializer -class CeleryTaskLogApi(LogTailApi): + +class CeleryTaskExecutionLogApi(LogTailApi): permission_classes = (IsValidUser,) task = None task_id = '' @@ -46,8 +48,8 @@ class CeleryTaskLogApi(LogTailApi): if new_path and os.path.isfile(new_path): return new_path try: - task = CeleryTask.objects.get(id=self.task_id) - except CeleryTask.DoesNotExist: + task = CeleryTaskExecution.objects.get(id=self.task_id) + except CeleryTaskExecution.DoesNotExist: return None return task.full_log_path @@ -94,3 +96,22 @@ class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet): queryset = super().get_queryset() queryset = queryset.exclude(description='') return queryset + + +class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): + queryset = CeleryTask.objects.all() + serializer_class = CeleryTaskSerializer + http_method_names = ('get', 'head', 'options',) + + +class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): + serializer_class = CeleryTaskExecutionSerializer + http_method_names = ('get', 'head', 'options',) + + def get_queryset(self): + task_id = self.kwargs.get("task_pk") + if task_id: + task = CeleryTask.objects.get(pk=task_id) + return CeleryTaskExecution.objects.filter(name=task.name) + else: + return CeleryTaskExecution.objects.none() diff --git a/apps/ops/migrations/0027_auto_20221024_1709.py b/apps/ops/migrations/0027_auto_20221024_1709.py new file mode 100644 index 000000000..4b29e4a3e --- /dev/null +++ b/apps/ops/migrations/0027_auto_20221024_1709.py @@ -0,0 +1,67 @@ +# Generated by Django 3.2.14 on 2022-10-24 09:09 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0026_auto_20221009_2050'), + ] + + operations = [ + migrations.CreateModel( + name='CeleryTaskExecution', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=1024)), + ('args', models.JSONField(verbose_name='Args')), + ('kwargs', models.JSONField(verbose_name='Kwargs')), + ('state', models.CharField(max_length=16, verbose_name='State')), + ('is_finished', models.BooleanField(default=False, verbose_name='Finished')), + ('date_published', models.DateTimeField(auto_now_add=True)), + ('date_start', models.DateTimeField(null=True)), + ('date_finished', models.DateTimeField(null=True)), + ], + ), + migrations.RenameField( + model_name='celerytask', + old_name='date_finished', + new_name='date_last_published', + ), + migrations.RemoveField( + model_name='celerytask', + name='args', + ), + migrations.RemoveField( + model_name='celerytask', + name='date_published', + ), + migrations.RemoveField( + model_name='celerytask', + name='date_start', + ), + migrations.RemoveField( + model_name='celerytask', + name='is_finished', + ), + migrations.RemoveField( + model_name='celerytask', + name='kwargs', + ), + migrations.RemoveField( + model_name='celerytask', + name='state', + ), + migrations.AddField( + model_name='celerytask', + name='description', + field=models.CharField(max_length=2048, null=True), + ), + migrations.AddField( + model_name='celerytask', + name='verbose_name', + field=models.CharField(max_length=1024, null=True), + ), + ] diff --git a/apps/ops/migrations/0028_auto_20221024_1712.py b/apps/ops/migrations/0028_auto_20221024_1712.py new file mode 100644 index 000000000..d246adbd8 --- /dev/null +++ b/apps/ops/migrations/0028_auto_20221024_1712.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.14 on 2022-10-24 09:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0027_auto_20221024_1709'), + ] + + operations = [ + migrations.RemoveField( + model_name='celerytask', + name='date_last_published', + ), + migrations.RemoveField( + model_name='celerytask', + name='description', + ), + migrations.RemoveField( + model_name='celerytask', + name='verbose_name', + ), + ] diff --git a/apps/ops/models/celery.py b/apps/ops/models/celery.py index 2291eb6f1..6ea4e2641 100644 --- a/apps/ops/models/celery.py +++ b/apps/ops/models/celery.py @@ -7,8 +7,27 @@ from django.utils.translation import gettext_lazy as _ from django.conf import settings from django.db import models +from ops.celery import app + class CeleryTask(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4) + name = models.CharField(max_length=1024) + + @property + def verbose_name(self): + task = app.tasks.get(self.name, None) + if task: + return getattr(task, 'verbose_name', None) + + @property + def description(self): + task = app.tasks.get(self.name, None) + if task: + return getattr(task, 'description', None) + + +class CeleryTaskExecution(models.Model): LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery') id = models.UUIDField(primary_key=True, default=uuid.uuid4) name = models.CharField(max_length=1024) diff --git a/apps/ops/serializers/celery.py b/apps/ops/serializers/celery.py index 8015c2482..b2aa6eb7a 100644 --- a/apps/ops/serializers/celery.py +++ b/apps/ops/serializers/celery.py @@ -5,10 +5,12 @@ from rest_framework import serializers from django_celery_beat.models import PeriodicTask __all__ = [ - 'CeleryResultSerializer', 'CeleryTaskSerializer', - 'CeleryPeriodTaskSerializer' + 'CeleryResultSerializer', 'CeleryTaskExecutionSerializer', + 'CeleryPeriodTaskSerializer', 'CeleryTaskSerializer' ] +from ops.models import CeleryTask, CeleryTaskExecution + class CeleryResultSerializer(serializers.Serializer): id = serializers.UUIDField() @@ -16,10 +18,6 @@ class CeleryResultSerializer(serializers.Serializer): state = serializers.CharField(max_length=16) -class CeleryTaskSerializer(serializers.Serializer): - pass - - class CeleryPeriodTaskSerializer(serializers.ModelSerializer): class Meta: model = PeriodicTask @@ -27,3 +25,19 @@ class CeleryPeriodTaskSerializer(serializers.ModelSerializer): 'name', 'task', 'enabled', 'description', 'last_run_at', 'total_run_count' ] + + +class CeleryTaskSerializer(serializers.ModelSerializer): + class Meta: + model = CeleryTask + fields = [ + 'id', 'name', 'verbose_name', 'description', + ] + + +class CeleryTaskExecutionSerializer(serializers.ModelSerializer): + class Meta: + model = CeleryTaskExecution + fields = [ + "id", "name", "args", "kwargs", "state", "is_finished", "date_published", "date_start", "date_finished" + ] diff --git a/apps/ops/signal_handlers.py b/apps/ops/signal_handlers.py index e48802d84..3cb9b3f70 100644 --- a/apps/ops/signal_handlers.py +++ b/apps/ops/signal_handlers.py @@ -1,12 +1,15 @@ import ast +from django.db import transaction +from django.dispatch import receiver from django.utils import translation, timezone from django.core.cache import cache -from celery import signals +from celery import signals, current_app from common.db.utils import close_old_connections, get_logger -from .models import CeleryTask - +from common.signals import django_ready +from .celery import app +from .models import CeleryTaskExecution, CeleryTask logger = get_logger(__name__) @@ -14,6 +17,21 @@ TASK_LANG_CACHE_KEY = 'TASK_LANG_{}' TASK_LANG_CACHE_TTL = 1800 +@receiver(django_ready) +def sync_registered_tasks(*args, **kwargs): + with transaction.atomic(): + db_tasks = CeleryTask.objects.all() + celery_task_names = [key for key in app.tasks] + db_task_names = [task.name for task in db_tasks] + + for task in db_tasks: + if task.name not in celery_task_names: + task.delete() + for task in celery_task_names: + if task not in db_task_names: + CeleryTask(name=task).save() + + @signals.before_task_publish.connect def before_task_publish(headers=None, **kwargs): task_id = headers.get('id') @@ -25,7 +43,7 @@ def before_task_publish(headers=None, **kwargs): @signals.task_prerun.connect def on_celery_task_pre_run(task_id='', **kwargs): # 更新状态 - CeleryTask.objects.filter(id=task_id).update(state='RUNNING', date_start=timezone.now()) + CeleryTaskExecution.objects.filter(id=task_id).update(state='RUNNING', date_start=timezone.now()) # 关闭之前的数据库连接 close_old_connections() @@ -41,7 +59,7 @@ def on_celery_task_post_run(task_id='', state='', **kwargs): close_old_connections() print("Task post run: ", task_id, state) - CeleryTask.objects.filter(id=task_id).update( + CeleryTaskExecution.objects.filter(id=task_id).update( state=state, date_finished=timezone.now(), is_finished=True ) @@ -72,4 +90,4 @@ def task_sent_handler(headers=None, body=None, **kwargs): 'args': args, 'kwargs': kwargs } - CeleryTask.objects.create(**data) + CeleryTaskExecution.objects.create(**data) diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index e9ba28eb7..dc3ac6e68 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -20,7 +20,7 @@ from .celery.utils import ( create_or_update_celery_periodic_tasks, get_celery_periodic_task, disable_celery_periodic_task, delete_celery_periodic_task ) -from .models import CeleryTask, AdHoc, Playbook +from .models import CeleryTaskExecution, AdHoc, Playbook from .notifications import ServerPerformanceCheckUtil logger = get_logger(__file__) @@ -94,9 +94,9 @@ def clean_celery_tasks_period(): logger.debug("Start clean celery task history") expire_days = get_log_keep_day('TASK_LOG_KEEP_DAYS') days_ago = timezone.now() - timezone.timedelta(days=expire_days) - tasks = CeleryTask.objects.filter(date_start__lt=days_ago) + tasks = CeleryTaskExecution.objects.filter(date_start__lt=days_ago) tasks.delete() - tasks = CeleryTask.objects.filter(date_start__isnull=True) + tasks = CeleryTaskExecution.objects.filter(date_start__isnull=True) tasks.delete() command = "find %s -mtime +%s -name '*.log' -type f -exec rm -f {} \\;" % ( settings.CELERY_LOG_DIR, expire_days diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index 49038b9b1..1edbd16e7 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -4,8 +4,9 @@ from __future__ import unicode_literals from django.urls import path from rest_framework.routers import DefaultRouter from rest_framework_bulk.routes import BulkRouter -from .. import api +from rest_framework_nested import routers +from .. import api app_name = "ops" @@ -16,12 +17,19 @@ router.register(r'adhoc', api.AdHocViewSet, 'adhoc') router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution') router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task') -urlpatterns = [ - path('celery/task//log/', api.CeleryTaskLogApi.as_view(), name='celery-task-log'), - path('celery/task//result/', api.CeleryResultApi.as_view(), name='celery-result'), +router.register(r'tasks', api.CeleryTaskViewSet, 'task') - path('ansible/task//log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'), +task_router = routers.NestedDefaultRouter(router, r'tasks', lookup='task') +task_router.register(r'executions', api.CeleryTaskExecutionViewSet, 'task-execution') + +urlpatterns = [ + + path('celery/task//task-execution//log/', api.CeleryTaskExecutionLogApi.as_view(), + name='celery-task-execution-log'), + path('celery/task//task-execution//result/', api.CeleryResultApi.as_view(), + name='celery-task-execution-result'), + + path('ansible/task-execution//log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'), ] -urlpatterns += router.urls -urlpatterns += bulk_router.urls +urlpatterns += (router.urls + bulk_router.urls + task_router.urls)