diff --git a/.dockerignore b/.dockerignore index 191381ee7..51451ada1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,7 @@ -.git \ No newline at end of file +.git +logs/* +data/* +.github +tmp/* +django.db +celerybeat.pid \ No newline at end of file diff --git a/.gitignore b/.gitignore index 04bf6bec7..38830de50 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ media celerybeat.pid django.db celerybeat-schedule.db +static diff --git a/apps/assets/api.py b/apps/assets/api.py index 064c2780e..8bcb67061 100644 --- a/apps/assets/api.py +++ b/apps/assets/api.py @@ -222,7 +222,7 @@ class AssetRefreshHardwareApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): asset_id = kwargs.get('pk') asset = get_object_or_404(Asset, pk=asset_id) - summary = update_assets_hardware_info_manual([asset]) + summary = update_assets_hardware_info_manual([asset])[1] if summary.get('dark'): return Response(summary['dark'].values(), status=501) else: @@ -255,7 +255,7 @@ class AdminUserTestConnectiveApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): admin_user = self.get_object() - test_admin_user_connectability_util.delay(admin_user, force=True) + test_admin_user_connectability_util.delay(admin_user) return Response({"msg": "Task created"}) @@ -268,7 +268,7 @@ class SystemUserPushApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): system_user = self.get_object() - push_system_user_to_cluster_assets_manual.delay(system_user, force=True) + push_system_user_to_cluster_assets_manual.delay(system_user) return Response({"msg": "Task created"}) @@ -281,5 +281,5 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): system_user = self.get_object() - test_system_user_connectability_manual.delay(system_user, force=True) + test_system_user_connectability_manual.delay(system_user) return Response({"msg": "Task created"}) diff --git a/apps/common/celery.py b/apps/common/celery.py index cbd54cb76..5b76978af 100644 --- a/apps/common/celery.py +++ b/apps/common/celery.py @@ -6,7 +6,7 @@ from functools import wraps from celery import Celery, subtask from celery.signals import worker_ready, worker_shutdown -from django.db.utils import ProgrammingError +from django.db.utils import ProgrammingError, OperationalError from .utils import get_logger @@ -46,7 +46,7 @@ def create_or_update_celery_periodic_tasks(tasks): crontab = None try: IntervalSchedule.objects.all().count() - except ProgrammingError: + except (ProgrammingError, OperationalError): return None if isinstance(detail.get("interval"), int): diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6ee4c6e6d..d4807ceba 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,5 @@ amqp==2.1.4 +six==1.11.0 ansible==2.4.2.0 asn1crypto==0.24.0 bcrypt==3.1.4 @@ -45,13 +46,11 @@ pycparser==2.18 pycrypto==2.6.1 pyldap==2.4.45 PyNaCl==1.2.1 -python-gssapi==0.6.4 pytz==2017.3 PyYAML==3.12 redis==2.10.6 requests==2.18.4 simplejson==3.13.2 -six==1.11.0 sshpubkeys==2.2.0 uritemplate==3.0.0 urllib3==1.22 @@ -59,3 +58,4 @@ vine==1.1.4 gunicorn==19.7.1 django_celery_beat==1.1.0 ephem==3.7.6.0 +python-gssapi==0.6.4 diff --git a/run_server.py b/run_server.py index 37f087b96..49e064e47 100644 --- a/run_server.py +++ b/run_server.py @@ -2,8 +2,10 @@ import os import subprocess +import threading import time -from threading import Thread +import argparse +import platform from apps import __version__ @@ -21,38 +23,16 @@ DEBUG = CONFIG.DEBUG LOG_LEVEL = CONFIG.LOG_LEVEL WORKERS = 4 +EXIT_EVENT = threading.Event() +EXIT_MSGS = [] + + try: os.makedirs(os.path.join(BASE_DIR, "data", "static")) os.makedirs(os.path.join(BASE_DIR, "data", "media")) except: pass -def start_gunicorn(): - print("# Start Gunicorn WSGI HTTP Server") - os.chdir(APPS_DIR) - cmd = "gunicorn jumpserver.wsgi -b {}:{} -w {}".format(HTTP_HOST, HTTP_PORT, WORKERS) - if DEBUG: - cmd += " --reload" - subprocess.call(cmd, shell=True) - - -def start_celery(): - print("- Start Celery as Distributed Task Queue") - os.chdir(APPS_DIR) - # Todo: Must set this environment, otherwise not no ansible result return - os.environ.setdefault('PYTHONOPTIMIZE', '1') - cmd = 'celery -A common worker -l {}'.format(LOG_LEVEL.lower()) - subprocess.call(cmd, shell=True) - - -def start_beat(): - print("- Start Beat as Periodic Task Scheduler") - os.chdir(APPS_DIR) - os.environ.setdefault('PYTHONOPTIMIZE', '1') - scheduler = "django_celery_beat.schedulers:DatabaseScheduler" - cmd = 'celery -A common beat -l {} --scheduler {} --max-interval 5 '.format(LOG_LEVEL, scheduler) - subprocess.call(cmd, shell=True) - def make_migrations(): print("Check database change, make migrations") @@ -60,28 +40,95 @@ def make_migrations(): subprocess.call('bash make_migrations.sh', shell=True) -def main(): +def collect_static(): + print("Collect static files") + os.chdir(os.path.join(BASE_DIR, 'apps')) + subprocess.call('python manage.py collectstatic --no-input', shell=True) + + +def start_gunicorn(): + print("- Start Gunicorn WSGI HTTP Server") + make_migrations() + collect_static() + os.chdir(APPS_DIR) + cmd = "gunicorn jumpserver.wsgi -b {}:{} -w {}".format(HTTP_HOST, HTTP_PORT, WORKERS) + if DEBUG: + cmd += " --reload" + subprocess.call(cmd, shell=True) + EXIT_MSGS.append("Gunicorn start failed") + EXIT_EVENT.set() + + +def start_celery(): + print("- Start Celery as Distributed Task Queue") + os.chdir(APPS_DIR) + # Todo: Must set this environment, otherwise not no ansible result return + os.environ.setdefault('PYTHONOPTIMIZE', '1') + + if platform.platform().startswith("Linux"): + cmd = """ + id jumpserver || useradd -s /sbin/nologin jumpserver; + su jumpserver -c 'celery -A common worker -l {}' + """.format(LOG_LEVEL.lower()) + else: + cmd = """ + export C_FORCE_ROOT=1;celery -A common worker -l {}' + """.format(LOG_LEVEL.lower()) + + subprocess.call(cmd, shell=True) + EXIT_MSGS.append("Celery start failed") + EXIT_EVENT.set() + + +def start_beat(): + print("- Start Beat as Periodic Task Scheduler") + os.chdir(APPS_DIR) + os.environ.setdefault('PYTHONOPTIMIZE', '1') + os.environ.setdefault('C_FORCE_ROOT', '1') + scheduler = "django_celery_beat.schedulers:DatabaseScheduler" + cmd = 'celery -A common beat -l {} --scheduler {} --max-interval 5 '.format(LOG_LEVEL, scheduler) + subprocess.call(cmd, shell=True) + EXIT_MSGS.append("Beat start failed") + EXIT_EVENT.set() + + +def start_service(services): make_migrations() print(time.ctime()) print('Jumpserver version {}, more see https://www.jumpserver.org'.format( __version__)) print('Quit the server with CONTROL-C.') + threads = [] - for func in (start_gunicorn, start_celery, start_beat): - t = Thread(target=func, args=()) - threads.append(t) - t.start() + if 'gunicorn' in args.services: + threads.append(threading.Thread(target=start_gunicorn, args=())) + if 'celery' in args.services: + threads.append(threading.Thread(target=start_celery, args=())) + if 'beat' in args.services: + threads.append(threading.Thread(target=start_beat, args=())) + if 'all' in args.services: + _threads = [] + for func in (start_gunicorn, start_celery, start_beat): + t = threading.Thread(target=func, args=()) + _threads.append(t) + threads = _threads for t in threads: - t.join() + t.start() + + if EXIT_EVENT.wait(): + print("\n\n" + "####" * 30) + print("\n".join(EXIT_MSGS)) if __name__ == '__main__': - main() - - - - + parser = argparse.ArgumentParser(description="Jumpserver start tools") + parser.add_argument("services", type=str, nargs='+', default="all", + choices=("all", "gunicorn", "celery", "beat"), + help="The service to start", + ) + args = parser.parse_args() + start_service(args.services)