perf: 增加定时任务清理僵尸作业 (#9714)

* perf: 增加定时任务清理僵尸作业

* perf: 优化代码结构

* fix: 修改翻译

* perf: 延长清理间隔时间

* perf: 添加优先级括号

---------

Co-authored-by: Aaron3S <chenyang@fit2cloud.com>
pull/9729/head
fit2bot 2023-02-23 17:01:24 +08:00 committed by GitHub
parent e1a1e838bf
commit b2f38c2218
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 91 additions and 63 deletions

View File

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-02-22 21:23+0800\n"
"POT-Creation-Date: 2023-02-23 16:29+0800\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -249,7 +249,7 @@ msgstr "アカウントバックアップ計画"
#: accounts/models/automations/backup_account.py:83
#: assets/models/automations/base.py:114 audits/models.py:55
#: ops/models/base.py:55 ops/models/celery.py:63 ops/models/job.py:186
#: ops/models/base.py:55 ops/models/celery.py:63 ops/models/job.py:191
#: perms/models/asset_permission.py:72 terminal/models/applet/host.py:109
#: terminal/models/session/session.py:45
#: tickets/models/ticket/apply_application.py:30
@ -371,7 +371,7 @@ msgstr "開始日"
#: accounts/models/automations/change_secret.py:74
#: assets/models/automations/base.py:115 ops/models/base.py:56
#: ops/models/celery.py:64 ops/models/job.py:187
#: ops/models/celery.py:64 ops/models/job.py:192
#: terminal/models/applet/host.py:110
msgid "Date finished"
msgstr "終了日"
@ -450,7 +450,7 @@ msgstr "アカウントの確認"
#: assets/serializers/platform.py:91 assets/serializers/platform.py:136
#: authentication/serializers/connect_token_secret.py:103 ops/mixin.py:21
#: ops/models/adhoc.py:21 ops/models/celery.py:15 ops/models/celery.py:57
#: ops/models/job.py:90 ops/models/playbook.py:23 ops/serializers/job.py:19
#: ops/models/job.py:91 ops/models/playbook.py:23 ops/serializers/job.py:19
#: orgs/models.py:69 perms/models/asset_permission.py:56 rbac/models/role.py:29
#: settings/models.py:33 settings/serializers/sms.py:6
#: terminal/models/applet/applet.py:27 terminal/models/component/endpoint.py:12
@ -546,7 +546,7 @@ msgstr "カテゴリ"
#: assets/models/cmd_filter.py:74 assets/models/platform.py:78
#: assets/serializers/asset/common.py:122 assets/serializers/platform.py:96
#: audits/serializers.py:48
#: authentication/serializers/connect_token_secret.py:116 ops/models/job.py:101
#: authentication/serializers/connect_token_secret.py:116 ops/models/job.py:102
#: perms/serializers/user_permission.py:26 terminal/models/applet/applet.py:31
#: terminal/models/component/storage.py:57
#: terminal/models/component/storage.py:146 terminal/serializers/applet.py:29
@ -599,7 +599,7 @@ msgstr "特別情報"
#: accounts/serializers/automations/base.py:22
#: assets/models/automations/base.py:19
#: assets/serializers/automations/base.py:20 ops/models/base.py:17
#: ops/models/job.py:103 ops/serializers/job.py:20
#: ops/models/job.py:104 ops/serializers/job.py:20
#: terminal/templates/terminal/_msg_command_execute_alert.html:16
msgid "Assets"
msgstr "資産"
@ -1065,7 +1065,7 @@ msgstr "SSHパブリックキー"
#: assets/models/_user.py:27 assets/models/cmd_filter.py:40
#: assets/models/cmd_filter.py:88 assets/models/group.py:23
#: common/db/models.py:37 ops/models/adhoc.py:27 ops/models/job.py:109
#: common/db/models.py:37 ops/models/adhoc.py:27 ops/models/job.py:110
#: ops/models/playbook.py:26 rbac/models/role.py:37 settings/models.py:38
#: terminal/models/applet/applet.py:36 terminal/models/applet/applet.py:158
#: terminal/models/applet/host.py:111 terminal/models/component/endpoint.py:24
@ -1078,7 +1078,7 @@ msgstr "コメント"
#: assets/models/_user.py:28 assets/models/automations/base.py:113
#: assets/models/cmd_filter.py:41 assets/models/group.py:22
#: common/db/models.py:35 ops/models/base.py:54 ops/models/job.py:185
#: common/db/models.py:35 ops/models/base.py:54 ops/models/job.py:190
#: users/models/user.py:943
msgid "Date created"
msgstr "作成された日付"
@ -1267,7 +1267,7 @@ msgid "Asset automation task"
msgstr "アセットの自動化タスク"
#: assets/models/automations/base.py:112 audits/models.py:177
#: audits/serializers.py:49 ops/models/base.py:49 ops/models/job.py:178
#: audits/serializers.py:49 ops/models/base.py:49 ops/models/job.py:183
#: terminal/models/applet/applet.py:157 terminal/models/applet/host.py:108
#: terminal/models/component/status.py:27 terminal/serializers/applet.py:18
#: terminal/serializers/applet_host.py:93 tickets/models/ticket/general.py:283
@ -3279,7 +3279,7 @@ msgstr "VCS"
msgid "Adhoc"
msgstr "コマンド#コマンド#"
#: ops/const.py:39 ops/models/job.py:99
#: ops/const.py:39 ops/models/job.py:100
msgid "Playbook"
msgstr "Playbook"
@ -3340,17 +3340,17 @@ msgstr "定期的または定期的に設定を行う必要があります"
msgid "Pattern"
msgstr "パターン"
#: ops/models/adhoc.py:24 ops/models/job.py:94
#: ops/models/adhoc.py:24 ops/models/job.py:95
msgid "Module"
msgstr "モジュール"
#: ops/models/adhoc.py:25 ops/models/celery.py:58 ops/models/job.py:93
#: ops/models/adhoc.py:25 ops/models/celery.py:58 ops/models/job.py:94
#: terminal/models/component/task.py:16
msgid "Args"
msgstr "アルグ"
#: ops/models/adhoc.py:26 ops/models/base.py:16 ops/models/base.py:53
#: ops/models/job.py:102 ops/models/job.py:184 ops/models/playbook.py:25
#: ops/models/job.py:103 ops/models/job.py:189 ops/models/playbook.py:25
#: terminal/models/session/sharing.py:23
msgid "Creator"
msgstr "作成者"
@ -3367,12 +3367,12 @@ msgstr "最後の実行"
msgid "Date last run"
msgstr "最終実行日"
#: ops/models/base.py:51 ops/models/job.py:182
#: ops/models/base.py:51 ops/models/job.py:187
#: xpack/plugins/cloud/models.py:170
msgid "Result"
msgstr "結果"
#: ops/models/base.py:52 ops/models/job.py:183
#: ops/models/base.py:52 ops/models/job.py:188
msgid "Summary"
msgstr "概要"
@ -3412,47 +3412,47 @@ msgstr "発売日"
msgid "Celery Task Execution"
msgstr "Celery タスク実行"
#: ops/models/job.py:96
#: ops/models/job.py:97
msgid "Chdir"
msgstr "実行ディレクトリ"
#: ops/models/job.py:97
#: ops/models/job.py:98
msgid "Timeout (Seconds)"
msgstr "タイムアウト(秒)"
#: ops/models/job.py:104
#: ops/models/job.py:105
msgid "Use Parameter Define"
msgstr "パラメータ定義を使用する"
#: ops/models/job.py:105
#: ops/models/job.py:106
msgid "Parameters define"
msgstr "パラメータ定義"
#: ops/models/job.py:106
#: ops/models/job.py:107
msgid "Runas"
msgstr "ユーザーとして実行"
#: ops/models/job.py:108
#: ops/models/job.py:109
msgid "Runas policy"
msgstr "ユーザー ポリシー"
#: ops/models/job.py:170
#: ops/models/job.py:171
msgid "Job"
msgstr "ジョブ#ジョブ#"
#: ops/models/job.py:181
#: ops/models/job.py:186
msgid "Parameters"
msgstr "パラメータ"
#: ops/models/job.py:189
#: ops/models/job.py:194
msgid "Material"
msgstr ""
#: ops/models/job.py:191
#: ops/models/job.py:196
msgid "Material Type"
msgstr ""
#: ops/models/job.py:442
#: ops/models/job.py:455
msgid "Job Execution"
msgstr "ジョブ実行"
@ -3528,6 +3528,10 @@ msgstr "定期的なタスクの作成または更新"
msgid "Periodic check service performance"
msgstr "サービスのパフォーマンスを定期的に確認する"
#: ops/tasks.py:114
msgid "Clean up unexpected jobs"
msgstr "例外ジョブのクリーンアップ"
#: ops/templates/ops/celery_task_log.html:4
msgid "Task log"
msgstr "タスクログ"
@ -5987,19 +5991,19 @@ msgstr ""
"チケットのタイトル: {} チケット申請者: {} チケットプロセッサ: {} チケットID: "
"{}"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "Change field"
msgstr "フィールドを変更"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "Before change"
msgstr "変更前"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "After change"
msgstr "変更後"
#: tickets/handlers/base.py:98
#: tickets/handlers/base.py:97
msgid "{} {} the ticket"
msgstr "{} {} チケット"
@ -6255,7 +6259,7 @@ msgstr "無効な承認アクション"
msgid "This user is not authorized to approve this ticket"
msgstr "このユーザーはこの作業指示を承認する権限がありません"
#: users/api/user.py:179
#: users/api/user.py:182
msgid "Could not reset self otp, use profile reset instead"
msgstr "自己otpをリセットできませんでした、代わりにプロファイルリセットを使用"

View File

@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: JumpServer 0.3.3\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-02-22 21:23+0800\n"
"POT-Creation-Date: 2023-02-23 16:29+0800\n"
"PO-Revision-Date: 2021-05-20 10:54+0800\n"
"Last-Translator: ibuler <ibuler@qq.com>\n"
"Language-Team: JumpServer team<ibuler@qq.com>\n"
@ -248,7 +248,7 @@ msgstr "账号备份计划"
#: accounts/models/automations/backup_account.py:83
#: assets/models/automations/base.py:114 audits/models.py:55
#: ops/models/base.py:55 ops/models/celery.py:63 ops/models/job.py:186
#: ops/models/base.py:55 ops/models/celery.py:63 ops/models/job.py:191
#: perms/models/asset_permission.py:72 terminal/models/applet/host.py:109
#: terminal/models/session/session.py:45
#: tickets/models/ticket/apply_application.py:30
@ -370,7 +370,7 @@ msgstr "开始日期"
#: accounts/models/automations/change_secret.py:74
#: assets/models/automations/base.py:115 ops/models/base.py:56
#: ops/models/celery.py:64 ops/models/job.py:187
#: ops/models/celery.py:64 ops/models/job.py:192
#: terminal/models/applet/host.py:110
msgid "Date finished"
msgstr "结束日期"
@ -449,7 +449,7 @@ msgstr "账号验证"
#: assets/serializers/platform.py:91 assets/serializers/platform.py:136
#: authentication/serializers/connect_token_secret.py:103 ops/mixin.py:21
#: ops/models/adhoc.py:21 ops/models/celery.py:15 ops/models/celery.py:57
#: ops/models/job.py:90 ops/models/playbook.py:23 ops/serializers/job.py:19
#: ops/models/job.py:91 ops/models/playbook.py:23 ops/serializers/job.py:19
#: orgs/models.py:69 perms/models/asset_permission.py:56 rbac/models/role.py:29
#: settings/models.py:33 settings/serializers/sms.py:6
#: terminal/models/applet/applet.py:27 terminal/models/component/endpoint.py:12
@ -542,7 +542,7 @@ msgstr "类别"
#: assets/models/cmd_filter.py:74 assets/models/platform.py:78
#: assets/serializers/asset/common.py:122 assets/serializers/platform.py:96
#: audits/serializers.py:48
#: authentication/serializers/connect_token_secret.py:116 ops/models/job.py:101
#: authentication/serializers/connect_token_secret.py:116 ops/models/job.py:102
#: perms/serializers/user_permission.py:26 terminal/models/applet/applet.py:31
#: terminal/models/component/storage.py:57
#: terminal/models/component/storage.py:146 terminal/serializers/applet.py:29
@ -595,7 +595,7 @@ msgstr "特殊信息"
#: accounts/serializers/automations/base.py:22
#: assets/models/automations/base.py:19
#: assets/serializers/automations/base.py:20 ops/models/base.py:17
#: ops/models/job.py:103 ops/serializers/job.py:20
#: ops/models/job.py:104 ops/serializers/job.py:20
#: terminal/templates/terminal/_msg_command_execute_alert.html:16
msgid "Assets"
msgstr "资产"
@ -1057,7 +1057,7 @@ msgstr "SSH公钥"
#: assets/models/_user.py:27 assets/models/cmd_filter.py:40
#: assets/models/cmd_filter.py:88 assets/models/group.py:23
#: common/db/models.py:37 ops/models/adhoc.py:27 ops/models/job.py:109
#: common/db/models.py:37 ops/models/adhoc.py:27 ops/models/job.py:110
#: ops/models/playbook.py:26 rbac/models/role.py:37 settings/models.py:38
#: terminal/models/applet/applet.py:36 terminal/models/applet/applet.py:158
#: terminal/models/applet/host.py:111 terminal/models/component/endpoint.py:24
@ -1070,7 +1070,7 @@ msgstr "备注"
#: assets/models/_user.py:28 assets/models/automations/base.py:113
#: assets/models/cmd_filter.py:41 assets/models/group.py:22
#: common/db/models.py:35 ops/models/base.py:54 ops/models/job.py:185
#: common/db/models.py:35 ops/models/base.py:54 ops/models/job.py:190
#: users/models/user.py:943
msgid "Date created"
msgstr "创建日期"
@ -1259,7 +1259,7 @@ msgid "Asset automation task"
msgstr "资产自动化任务"
#: assets/models/automations/base.py:112 audits/models.py:177
#: audits/serializers.py:49 ops/models/base.py:49 ops/models/job.py:178
#: audits/serializers.py:49 ops/models/base.py:49 ops/models/job.py:183
#: terminal/models/applet/applet.py:157 terminal/models/applet/host.py:108
#: terminal/models/component/status.py:27 terminal/serializers/applet.py:18
#: terminal/serializers/applet_host.py:93 tickets/models/ticket/general.py:283
@ -3244,7 +3244,7 @@ msgstr "VCS"
msgid "Adhoc"
msgstr "命令"
#: ops/const.py:39 ops/models/job.py:99
#: ops/const.py:39 ops/models/job.py:100
msgid "Playbook"
msgstr "Playbook"
@ -3305,17 +3305,17 @@ msgstr "需要周期或定期设置"
msgid "Pattern"
msgstr "模式"
#: ops/models/adhoc.py:24 ops/models/job.py:94
#: ops/models/adhoc.py:24 ops/models/job.py:95
msgid "Module"
msgstr "模块"
#: ops/models/adhoc.py:25 ops/models/celery.py:58 ops/models/job.py:93
#: ops/models/adhoc.py:25 ops/models/celery.py:58 ops/models/job.py:94
#: terminal/models/component/task.py:16
msgid "Args"
msgstr "参数"
#: ops/models/adhoc.py:26 ops/models/base.py:16 ops/models/base.py:53
#: ops/models/job.py:102 ops/models/job.py:184 ops/models/playbook.py:25
#: ops/models/job.py:103 ops/models/job.py:189 ops/models/playbook.py:25
#: terminal/models/session/sharing.py:23
msgid "Creator"
msgstr "创建者"
@ -3332,12 +3332,12 @@ msgstr "最后执行"
msgid "Date last run"
msgstr "最后运行日期"
#: ops/models/base.py:51 ops/models/job.py:182
#: ops/models/base.py:51 ops/models/job.py:187
#: xpack/plugins/cloud/models.py:170
msgid "Result"
msgstr "结果"
#: ops/models/base.py:52 ops/models/job.py:183
#: ops/models/base.py:52 ops/models/job.py:188
msgid "Summary"
msgstr "汇总"
@ -3377,47 +3377,47 @@ msgstr "发布日期"
msgid "Celery Task Execution"
msgstr "Celery 任务执行"
#: ops/models/job.py:96
#: ops/models/job.py:97
msgid "Chdir"
msgstr "运行目录"
#: ops/models/job.py:97
#: ops/models/job.py:98
msgid "Timeout (Seconds)"
msgstr "超时时间(秒)"
#: ops/models/job.py:104
#: ops/models/job.py:105
msgid "Use Parameter Define"
msgstr "使用参数定义"
#: ops/models/job.py:105
#: ops/models/job.py:106
msgid "Parameters define"
msgstr "参数定义"
#: ops/models/job.py:106
#: ops/models/job.py:107
msgid "Runas"
msgstr "运行用户"
#: ops/models/job.py:108
#: ops/models/job.py:109
msgid "Runas policy"
msgstr "用户策略"
#: ops/models/job.py:170
#: ops/models/job.py:171
msgid "Job"
msgstr "作业"
#: ops/models/job.py:181
#: ops/models/job.py:186
msgid "Parameters"
msgstr "参数"
#: ops/models/job.py:189
#: ops/models/job.py:194
msgid "Material"
msgstr ""
#: ops/models/job.py:191
#: ops/models/job.py:196
msgid "Material Type"
msgstr ""
#: ops/models/job.py:442
#: ops/models/job.py:455
msgid "Job Execution"
msgstr "作业执行"
@ -3493,6 +3493,10 @@ msgstr "创建或更新周期任务"
msgid "Periodic check service performance"
msgstr "周期检测服务性能"
#: ops/tasks.py:114
msgid "Clean up unexpected jobs"
msgstr "清理异常作业"
#: ops/templates/ops/celery_task_log.html:4
msgid "Task log"
msgstr "任务列表"
@ -5909,19 +5913,19 @@ msgid ""
msgstr ""
"通过工单创建, 工单标题: {}, 工单申请人: {}, 工单处理人: {}, 工单 ID: {}"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "Change field"
msgstr "变更字段"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "Before change"
msgstr "变更前"
#: tickets/handlers/base.py:86
#: tickets/handlers/base.py:85
msgid "After change"
msgstr "变更后"
#: tickets/handlers/base.py:98
#: tickets/handlers/base.py:97
msgid "{} {} the ticket"
msgstr "{} {} 工单"
@ -6175,7 +6179,7 @@ msgstr "无效的审批动作"
msgid "This user is not authorized to approve this ticket"
msgstr "此用户无权审批此工单"
#: users/api/user.py:179
#: users/api/user.py:182
msgid "Could not reset self otp, use profile reset instead"
msgstr "不能在该页面重置 MFA 多因子认证, 请去个人信息页面重置"

View File

@ -3,6 +3,7 @@ import logging
import os
import uuid
from collections import defaultdict
from datetime import timedelta
from celery import current_task
from django.conf import settings
@ -172,6 +173,10 @@ class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
ordering = ['date_created']
zombie_task_exception = Exception(
'This task has been marked as a zombie task because it has not updated its status for too long')
class JobExecution(JMSOrgBaseModel):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
task_id = models.UUIDField(null=True)
@ -190,6 +195,14 @@ class JobExecution(JMSOrgBaseModel):
job_type = models.CharField(max_length=128, choices=Types.choices, default=Types.adhoc,
verbose_name=_("Material Type"))
# clean up zombie execution
@classmethod
def clean_unexpected_execution(cls):
for execution in cls.objects.filter(status__in=[JobStatus.running]).all():
if execution.date_created < (timezone.now() - timedelta(hours=3)):
execution.set_error(zombie_task_exception)
@property
def current_job(self):
if self.job.version != self.job_version:

View File

@ -5,7 +5,7 @@ from celery.exceptions import SoftTimeLimitExceeded
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger, get_object_or_none
from orgs.utils import tmp_to_org
from orgs.utils import tmp_to_org, tmp_to_root_org
from .celery.decorator import (
register_as_period_task, after_app_ready_start
)
@ -109,3 +109,10 @@ def create_or_update_registered_periodic_tasks():
@register_as_period_task(interval=3600)
def check_server_performance_period():
ServerPerformanceCheckUtil().check_and_publish()
@shared_task(verbose_name=_("Clean up unexpected jobs"))
@register_as_period_task(interval=3600)
def clean_up_unexpected_jobs():
with tmp_to_root_org():
JobExecution.clean_unexpected_execution()