github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

fix: merge v3

pull/9075/head
Jiangjie.Bai 2022-11-15 17:50:44 +08:00
parent 3e31c9ed7f ae0ede5f73
commit aa9e5d2432
21 changed files with 1730 additions and 1448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Dockerfile
View File

@ -1,53 +1,66 @@
FROM python:3.8-slim as stage-build
ARG TARGETARCH
ARG VERSION
ENV VERSION=$VERSION
WORKDIR /opt/jumpserver
ADD . .
RUN cd utils && bash -ixeu build.sh
FROM python:3.8-slim
ARG TARGETARCH
MAINTAINER JumpServer Team <ibuler@qq.com>
ARG BUILD_DEPENDENCIES=" \
g++ \
make \
pkg-config"
g++ \
make \
pkg-config"
ARG DEPENDENCIES=" \
default-libmysqlclient-dev \
freetds-dev \
libpq-dev \
libffi-dev \
libldap2-dev \
libsasl2-dev \
libxml2-dev \
libxmlsec1-dev \
libxmlsec1-openssl \
libaio-dev \
openssh-client \
sshpass"
default-libmysqlclient-dev \
freetds-dev \
libpq-dev \
libffi-dev \
libjpeg-dev \
libldap2-dev \
libsasl2-dev \
libxml2-dev \
libxmlsec1-dev \
libxmlsec1-openssl \
libaio-dev \
openssh-client \
sshpass"
ARG TOOLS=" \
curl \
default-mysql-client \
iproute2 \
iputils-ping \
locales \
procps \
redis-tools \
telnet \
vim \
unzip \
wget"
ca-certificates \
curl \
default-mysql-client \
iputils-ping \
locales \
procps \
redis-tools \
telnet \
vim \
unzip \
wget"
RUN sed -i 's@http://.*.debian.org@http://mirrors.ustc.edu.cn@g' /etc/apt/sources.list \
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
sed -i 's@http://.*.debian.org@http://mirrors.ustc.edu.cn@g' /etc/apt/sources.list \
&& rm -f /etc/apt/apt.conf.d/docker-clean \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& apt-get update \
&& apt-get -y install --no-install-recommends ${BUILD_DEPENDENCIES} \
&& apt-get -y install --no-install-recommends ${DEPENDENCIES} \
&& apt-get -y install --no-install-recommends ${TOOLS} \
&& localedef -c -f UTF-8 -i zh_CN zh_CN.UTF-8 \
&& cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& mkdir -p /root/.ssh/ \
&& echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \
&& sed -i "s@# alias l@alias l@g" ~/.bashrc \
&& echo "set mouse-=a" > ~/.vimrc \
&& echo "no" | dpkg-reconfigure dash \
&& echo "zh_CN.UTF-8" | dpkg-reconfigure locales \
&& rm -rf /var/lib/apt/lists/*
ARG TARGETARCH
ARG ORACLE_LIB_MAJOR=19
ARG ORACLE_LIB_MINOR=10
ENV ORACLE_FILE="instantclient-basiclite-linux.${TARGETARCH:-amd64}-${ORACLE_LIB_MAJOR}.${ORACLE_LIB_MINOR}.0.0.0dbru.zip"
@ -68,21 +81,18 @@ ARG PIP_MIRROR=https://pypi.douban.com/simple
ENV PIP_MIRROR=$PIP_MIRROR
ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
# 因为以 jms 或者 jumpserver 开头的 mirror 上可能没有
RUN pip install --upgrade pip==20.2.4 setuptools==49.6.0 wheel==0.34.2 -i ${PIP_MIRROR} \
&& pip install --no-cache-dir $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
&& pip install --no-cache-dir -r requirements/requirements.txt -i ${PIP_MIRROR} \
&& rm -rf ~/.cache/pip
ARG VERSION
ENV VERSION=$VERSION
RUN --mount=type=cache,target=/root/.cache/pip \
set -ex \
&& pip config set global.index-url ${PIP_MIRROR} \
&& pip install --upgrade pip \
&& pip install --upgrade setuptools wheel \
&& pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
&& pip install -r requirements/requirements.txt
ADD . .
RUN cd utils \
&& bash -ixeu build.sh \
&& mv ../release/jumpserver /opt/jumpserver \
&& rm -rf /tmp/build \
&& echo > /opt/jumpserver/config.yml
COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
RUN echo > /opt/jumpserver/config.yml \
&& rm -rf /tmp/build
WORKDIR /opt/jumpserver
VOLUME /opt/jumpserver/data

3
apps/assets/tasks/automation.py
View File

@ -1,4 +1,5 @@
from celery import shared_task
from django.utils.translation import gettext_lazy as _
from orgs.utils import tmp_to_root_org, tmp_to_org
from common.utils import get_logger, get_object_or_none
@ -7,7 +8,7 @@ from assets.const import AutomationTypes
logger = get_logger(__file__)
@shared_task(queue='ansible')
@shared_task(queue='ansible', verbose_name=_('Execute automation'))
def execute_automation(pid, trigger, tp):
model = AutomationTypes.get_type_model(tp)
with tmp_to_root_org():

3
apps/assets/tasks/backup.py
View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
#
from celery import shared_task
from django.utils.translation import gettext_lazy as _
from common.utils import get_object_or_none, get_logger
from orgs.utils import tmp_to_org, tmp_to_root_org
@ -9,7 +10,7 @@ from assets.models import AccountBackupPlan
logger = get_logger(__file__)
@shared_task
@shared_task(verbose_name=_('Execute account backup plan'))
def execute_account_backup_plan(pid, trigger):
with tmp_to_root_org():
plan = get_object_or_none(AccountBackupPlan, pk=pid)

3
apps/assets/tasks/gather_accounts.py
View File

@ -1,6 +1,7 @@
# ~*~ coding: utf-8 ~*~
from celery import shared_task
from django.utils.translation import gettext_noop
from django.utils.translation import gettext_lazy as _
from orgs.utils import tmp_to_root_org, org_aware_func
from common.utils import get_logger
@ -24,7 +25,7 @@ def gather_asset_accounts_util(nodes, task_name):
instance.execute()
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Gather asset accounts'))
def gather_asset_accounts(node_ids, task_name=None):
if task_name is None:
task_name = gettext_noop("Gather assets accounts")

5
apps/assets/tasks/gather_facts.py
View File

@ -2,6 +2,7 @@
#
from celery import shared_task
from django.utils.translation import gettext_noop
from django.utils.translation import gettext_lazy as _
from common.utils import get_logger
from orgs.utils import org_aware_func, tmp_to_root_org
@ -40,7 +41,7 @@ def update_assets_hardware_info_util(assets=None, nodes=None, task_name=None):
instance.execute()
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets'))
def update_assets_hardware_info_manual(asset_ids):
from assets.models import Asset
with tmp_to_root_org():
@ -49,7 +50,7 @@ def update_assets_hardware_info_manual(asset_ids):
update_assets_hardware_info_util(assets=assets, task_name=task_name)
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets under a node'))
def update_node_assets_hardware_info_manual(node_id):
from assets.models import Node
with tmp_to_root_org():

5
apps/assets/tasks/nodes_amount.py
View File

@ -10,11 +10,10 @@ from common.utils.lock import AcquireFailed
from common.utils import get_logger
from common.const.crontab import CRONTAB_AT_AM_TWO
logger = get_logger(__file__)
@shared_task
@shared_task(verbose_name=_('Check the amount of assets under the node'))
def check_node_assets_amount_task(org_id=None):
if org_id is None:
orgs = Organization.objects.all()
@ -32,6 +31,6 @@ def check_node_assets_amount_task(org_id=None):
@register_as_period_task(crontab=CRONTAB_AT_AM_TWO)
@shared_task
@shared_task(verbose_name=_('Periodic check the amount of assets under the node'))
def check_node_assets_amount_period_task():
check_node_assets_amount_task()

5
apps/assets/tasks/ping.py
View File

@ -1,6 +1,7 @@
# ~*~ coding: utf-8 ~*~
from celery import shared_task
from django.utils.translation import gettext_noop
from django.utils.translation import gettext_lazy as _
from common.utils import get_logger
from orgs.utils import org_aware_func, tmp_to_root_org
@ -29,7 +30,7 @@ def test_asset_connectivity_util(assets, task_name=None):
instance.execute()
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of a asset'))
def test_assets_connectivity_manual(asset_ids):
from assets.models import Asset
with tmp_to_root_org():
@ -39,7 +40,7 @@ def test_assets_connectivity_manual(asset_ids):
test_asset_connectivity_util(assets, task_name=task_name)
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of assets under a node'))
def test_node_assets_connectivity_manual(node_id):
from assets.models import Node
with tmp_to_root_org():

3
apps/assets/tasks/push_account.py
View File

@ -3,6 +3,7 @@ from django.utils.translation import gettext_noop
from common.utils import get_logger
from orgs.utils import org_aware_func, tmp_to_root_org
from django.utils.translation import ugettext_lazy as _
logger = get_logger(__file__)
__all__ = [
@ -27,7 +28,7 @@ def push_accounts_to_assets_util(accounts, assets):
instance.execute()
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Push accounts to assets'))
def push_accounts_to_assets(account_ids, asset_ids):
from assets.models import Asset, Account
with tmp_to_root_org():

3
apps/assets/tasks/verify_account.py
View File

@ -1,5 +1,6 @@
from celery import shared_task
from django.utils.translation import gettext_noop
from django.utils.translation import ugettext as _
from common.utils import get_logger
from orgs.utils import org_aware_func, tmp_to_root_org
@ -26,7 +27,7 @@ def verify_accounts_connectivity_util(accounts, assets, task_name):
instance.execute()
@shared_task(queue="ansible")
@shared_task(queue="ansible", verbose_name=_('Verify asset account availability'))
def verify_accounts_connectivity(account_ids, asset_ids):
from assets.models import Asset, Account
with tmp_to_root_org():

5
apps/common/tasks.py
View File

@ -1,5 +1,6 @@
import os
from django.utils.translation import ugettext_lazy as _
from django.core.mail import send_mail, EmailMultiAlternatives
from django.conf import settings
from celery import shared_task
@ -9,7 +10,7 @@ from .utils import get_logger
logger = get_logger(__file__)
@shared_task
@shared_task(verbose_name=_("Send email"))
def send_mail_async(*args, **kwargs):
""" Using celery to send email async
@ -36,7 +37,7 @@ def send_mail_async(*args, **kwargs):
logger.error("Sending mail error: {}".format(e))
@shared_task
@shared_task(verbose_name=_("Send email attachment"))
def send_mail_attachment_async(subject, message, recipient_list, attachment_list=None):
if attachment_list is None:
attachment_list = []

4
apps/locale/ja/LC_MESSAGES/django.mo
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:07f1cfd07039142f4847b4139586bf815467f266119eae57476c073130f0ac92
size 118098
oid sha256:0b54b29587fa79fd51a8e1836eba016c2a64419dc0981bac65daa356f6e180f2
size 117154

1455
apps/locale/ja/LC_MESSAGES/django.po
View File

File diff suppressed because it is too large Load Diff

4
apps/locale/zh/LC_MESSAGES/django.mo
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:314c29cb8b10aaddbb030bf49af293be23f0153ff1f1c7562946879574ce6de8
size 102801
oid sha256:bf423289503715e2a574bce56bf6b1b323e0355ef18dbd1c8de37c66c0fb5b25
size 104080

1451
apps/locale/zh/LC_MESSAGES/django.po
View File

File diff suppressed because it is too large Load Diff

3
apps/notifications/notifications.py
View File

@ -16,7 +16,6 @@ from .models import SystemMsgSubscription, UserMsgSubscription
__all__ = ('SystemMessage', 'UserMessage', 'system_msgs', 'Message')
system_msgs = []
user_msgs = []
@ -44,7 +43,7 @@ class MessageType(type):
return clz
@shared_task
@shared_task(verbose_name=_('Publish the station message'))
def publish_task(msg):
msg.publish()

4
apps/ops/api/celery.py
View File

@ -104,10 +104,12 @@ class CelerySummaryAPIView(generics.RetrieveAPIView):
class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
queryset = CeleryTask.objects.all()
serializer_class = CeleryTaskSerializer
http_method_names = ('get', 'head', 'options',)
def get_queryset(self):
return CeleryTask.objects.exclude(name__startswith='celery')
class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
serializer_class = CeleryTaskExecutionSerializer

3
apps/ops/models/celery.py
View File

@ -37,6 +37,9 @@ class CeleryTask(models.Model):
return "yellow"
return "green"
class Meta:
ordering = ('name',)
class CeleryTaskExecution(models.Model):
LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery')

2
apps/ops/models/playbook.py
View File

@ -16,4 +16,4 @@ class Playbook(BaseCreateUpdateModel):
@property
def work_path(self):
return os.path.join(settings.DATA_DIR, "ops", "playbook", self.id.__str__())
return os.path.join(settings.DATA_DIR, "ops", "playbook", self.id.__str__(), "main.yaml")

114
apps/ops/tasks.py
View File

@ -1,19 +1,15 @@
# coding: utf-8
import os
import random
import subprocess
import time
from django.conf import settings
from celery import shared_task, subtask
from celery import signals
from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, gettext
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger, get_object_or_none, get_log_keep_day
from orgs.utils import tmp_to_root_org, tmp_to_org
from .celery.decorator import (
register_as_period_task, after_app_shutdown_clean_periodic,
after_app_ready_start
@ -22,16 +18,12 @@ from .celery.utils import (
create_or_update_celery_periodic_tasks, get_celery_periodic_task,
disable_celery_periodic_task, delete_celery_periodic_task
)
from .models import CeleryTaskExecution, Playbook, Job, JobExecution
from .models import CeleryTaskExecution, Job, JobExecution
from .notifications import ServerPerformanceCheckUtil
logger = get_logger(__file__)
def rerun_task():
pass
@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
def run_ops_job(job_id, **kwargs):
job = get_object_or_none(Job, id=job_id)
@ -59,64 +51,7 @@ def run_ops_job_executions(execution_id, **kwargs):
logger.error("Start adhoc execution error: {}".format(e))
@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
def run_adhoc(tid, **kwargs):
"""
:param tid: is the tasks serialized data
:param callback: callback function name
:return:
"""
with tmp_to_root_org():
task = get_object_or_none(AdHoc, id=tid)
if not task:
logger.error("No task found")
return
with tmp_to_org(task.org):
execution = task.create_execution()
try:
execution.start(**kwargs)
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run adhoc timeout")
except Exception as e:
execution.set_error(e)
logger.error("Start adhoc execution error: {}".format(e))
@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible command"))
def run_playbook(pid, **kwargs):
with tmp_to_root_org():
task = get_object_or_none(Playbook, id=pid)
if not task:
logger.error("No task found")
return
with tmp_to_org(task.org):
execution = task.create_execution()
try:
execution.start(**kwargs)
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run playbook timeout")
except Exception as e:
execution.set_error(e)
logger.error("Run playbook execution error: {}".format(e))
@shared_task
@after_app_shutdown_clean_periodic
@register_as_period_task(interval=3600 * 24, description=_("Clean task history period"))
def clean_tasks_adhoc_period():
logger.debug("Start clean task adhoc and run history")
tasks = Task.objects.all()
for task in tasks:
adhoc = task.adhoc.all().order_by('-date_created')[5:]
for ad in adhoc:
ad.execution.all().delete()
ad.delete()
@shared_task
@shared_task(verbose_name=_('Periodic clear celery tasks'))
@after_app_shutdown_clean_periodic
@register_as_period_task(interval=3600 * 24, description=_("Clean celery log period"))
def clean_celery_tasks_period():
@ -135,7 +70,7 @@ def clean_celery_tasks_period():
subprocess.call(command, shell=True)
@shared_task
@shared_task(verbose_name=_('Clear celery periodic tasks'))
@after_app_ready_start
def clean_celery_periodic_tasks():
"""清除celery定时任务"""
@ -158,7 +93,7 @@ def clean_celery_periodic_tasks():
logger.info('Clean task failure: {}'.format(task))
@shared_task
@shared_task(verbose_name=_('Create or update periodic tasks'))
@after_app_ready_start
def create_or_update_registered_periodic_tasks():
from .celery.decorator import get_register_period_tasks
@ -166,42 +101,7 @@ def create_or_update_registered_periodic_tasks():
create_or_update_celery_periodic_tasks(task)
@shared_task
@shared_task(verbose_name=_("Periodic check service performance"))
@register_as_period_task(interval=3600)
def check_server_performance_period():
ServerPerformanceCheckUtil().check_and_publish()
@shared_task(verbose_name=_("Hello"), comment="an test shared task")
def hello(name, callback=None):
from users.models import User
import time
count = User.objects.count()
print(gettext("Hello") + ': ' + name)
print("Count: ", count)
time.sleep(1)
return gettext("Hello")
@shared_task(verbose_name=_("Hello Error"), comment="an test shared task error")
def hello_error():
raise Exception("must be error")
@shared_task(verbose_name=_("Hello Random"), comment="some time error and some time success")
def hello_random():
i = random.randint(0, 1)
if i == 1:
raise Exception("must be error")
@shared_task(verbose_name="Hello Running", comment="an task running 1m")
def hello_running(sec=60):
time.sleep(sec)
@shared_task
def hello_callback(result):
print(result)
print("Hello callback")

3
apps/orgs/tasks.py
View File

@ -1,11 +1,12 @@
from celery import shared_task
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
logger = get_logger(__file__)
@shared_task
@shared_task(verbose_name=_("Refresh organization cache"))
def refresh_org_cache_task(*fields):
from .caches import OrgResourceStatisticsCache
OrgResourceStatisticsCache.refresh(*fields)

4
entrypoint.sh
View File

@ -11,6 +11,9 @@ action="${1-start}"
service="${2-all}"
trap cleanup EXIT
rm -f /opt/jumpserver/tmp/*.pid
if [[ "$action" == "bash" || "$action" == "sh" ]];then
bash
elif [[ "$action" == "sleep" ]];then
@ -19,4 +22,3 @@ elif [[ "$action" == "sleep" ]];then
else
python jms "${action}" "${service}"
fi