[Docker] For docker update

.dockerignore

@@ -1 +1,7 @@
 .git
+logs/*
+data/*
+.github
+tmp/*
+django.db
+celerybeat.pid

.gitignore

@@ -29,3 +29,4 @@ media
 celerybeat.pid
 django.db
 celerybeat-schedule.db
+static

apps/assets/api.py

@@ -222,7 +222,7 @@ class AssetRefreshHardwareApi(generics.RetrieveAPIView):
     def retrieve(self, request, *args, **kwargs):
         asset_id = kwargs.get('pk')
         asset = get_object_or_404(Asset, pk=asset_id)
-        summary = update_assets_hardware_info_manual([asset])
+        summary = update_assets_hardware_info_manual([asset])[1]
         if summary.get('dark'):
             return Response(summary['dark'].values(), status=501)
         else:
@@ -255,7 +255,7 @@ class AdminUserTestConnectiveApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         admin_user = self.get_object()
-        test_admin_user_connectability_util.delay(admin_user, force=True)
+        test_admin_user_connectability_util.delay(admin_user)
         return Response({"msg": "Task created"})
 
 
@@ -268,7 +268,7 @@ class SystemUserPushApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
-        push_system_user_to_cluster_assets_manual.delay(system_user, force=True)
+        push_system_user_to_cluster_assets_manual.delay(system_user)
         return Response({"msg": "Task created"})
 
 
@@ -281,5 +281,5 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
-        test_system_user_connectability_manual.delay(system_user, force=True)
+        test_system_user_connectability_manual.delay(system_user)
         return Response({"msg": "Task created"})

apps/common/celery.py

@@ -6,7 +6,7 @@ from functools import wraps
 
 from celery import Celery, subtask
 from celery.signals import worker_ready, worker_shutdown
-from django.db.utils import ProgrammingError
+from django.db.utils import ProgrammingError, OperationalError
 
 from .utils import get_logger
 
@@ -46,7 +46,7 @@ def create_or_update_celery_periodic_tasks(tasks):
         crontab = None
         try:
             IntervalSchedule.objects.all().count()
-        except ProgrammingError:
+        except (ProgrammingError, OperationalError):
             return None
 
         if isinstance(detail.get("interval"), int):

requirements/requirements.txt

@@ -1,4 +1,5 @@
 amqp==2.1.4
+six==1.11.0
 ansible==2.4.2.0
 asn1crypto==0.24.0
 bcrypt==3.1.4
@@ -45,13 +46,11 @@ pycparser==2.18
 pycrypto==2.6.1
 pyldap==2.4.45
 PyNaCl==1.2.1
-python-gssapi==0.6.4
 pytz==2017.3
 PyYAML==3.12
 redis==2.10.6
 requests==2.18.4
 simplejson==3.13.2
-six==1.11.0
 sshpubkeys==2.2.0
 uritemplate==3.0.0
 urllib3==1.22
@@ -59,3 +58,4 @@ vine==1.1.4
 gunicorn==19.7.1
 django_celery_beat==1.1.0
 ephem==3.7.6.0
+python-gssapi==0.6.4

run_server.py

@@ -2,8 +2,10 @@
 
 import os
 import subprocess
+import threading
 import time
-from threading import Thread
+import argparse
+import platform
 
 from apps import __version__
 
@@ -21,38 +23,16 @@ DEBUG = CONFIG.DEBUG
 LOG_LEVEL = CONFIG.LOG_LEVEL
 WORKERS = 4
 
+EXIT_EVENT = threading.Event()
+EXIT_MSGS = []
+
+
 try:
     os.makedirs(os.path.join(BASE_DIR, "data", "static"))
     os.makedirs(os.path.join(BASE_DIR, "data", "media"))
 except:
     pass
 
-def start_gunicorn():
-    print("# Start Gunicorn WSGI HTTP Server")
-    os.chdir(APPS_DIR)
-    cmd = "gunicorn jumpserver.wsgi -b {}:{} -w {}".format(HTTP_HOST, HTTP_PORT, WORKERS)
-    if DEBUG:
-        cmd += " --reload"
-    subprocess.call(cmd, shell=True)
-
-
-def start_celery():
-    print("- Start Celery as Distributed Task Queue")
-    os.chdir(APPS_DIR)
-    # Todo: Must set this environment, otherwise not no ansible result return
-    os.environ.setdefault('PYTHONOPTIMIZE', '1')
-    cmd = 'celery -A common worker -l {}'.format(LOG_LEVEL.lower())
-    subprocess.call(cmd, shell=True)
-
-
-def start_beat():
-    print("- Start Beat as Periodic Task Scheduler")
-    os.chdir(APPS_DIR)
-    os.environ.setdefault('PYTHONOPTIMIZE', '1')
-    scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
-    cmd = 'celery -A common beat  -l {} --scheduler {} --max-interval 5 '.format(LOG_LEVEL, scheduler)
-    subprocess.call(cmd, shell=True)
-
 
 def make_migrations():
     print("Check database change, make migrations")
@@ -60,28 +40,95 @@ def make_migrations():
     subprocess.call('bash make_migrations.sh', shell=True)
 
 
-def main():
+def collect_static():
+    print("Collect static files")
+    os.chdir(os.path.join(BASE_DIR, 'apps'))
+    subprocess.call('python manage.py collectstatic --no-input', shell=True)
+
+
+def start_gunicorn():
+    print("- Start Gunicorn WSGI HTTP Server")
+    make_migrations()
+    collect_static()
+    os.chdir(APPS_DIR)
+    cmd = "gunicorn jumpserver.wsgi -b {}:{} -w {}".format(HTTP_HOST, HTTP_PORT, WORKERS)
+    if DEBUG:
+        cmd += " --reload"
+    subprocess.call(cmd, shell=True)
+    EXIT_MSGS.append("Gunicorn start failed")
+    EXIT_EVENT.set()
+
+
+def start_celery():
+    print("- Start Celery as Distributed Task Queue")
+    os.chdir(APPS_DIR)
+    # Todo: Must set this environment, otherwise not no ansible result return
+    os.environ.setdefault('PYTHONOPTIMIZE', '1')
+
+    if platform.platform().startswith("Linux"):
+        cmd = """
+        id jumpserver || useradd -s /sbin/nologin jumpserver;
+        su jumpserver -c 'celery -A common worker -l {}'
+        """.format(LOG_LEVEL.lower())
+    else:
+        cmd = """
+        export C_FORCE_ROOT=1;celery -A common worker -l {}'
+        """.format(LOG_LEVEL.lower())
+
+    subprocess.call(cmd, shell=True)
+    EXIT_MSGS.append("Celery start failed")
+    EXIT_EVENT.set()
+
+
+def start_beat():
+    print("- Start Beat as Periodic Task Scheduler")
+    os.chdir(APPS_DIR)
+    os.environ.setdefault('PYTHONOPTIMIZE', '1')
+    os.environ.setdefault('C_FORCE_ROOT', '1')
+    scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
+    cmd = 'celery -A common beat  -l {} --scheduler {} --max-interval 5 '.format(LOG_LEVEL, scheduler)
+    subprocess.call(cmd, shell=True)
+    EXIT_MSGS.append("Beat start failed")
+    EXIT_EVENT.set()
+
+
+def start_service(services):
     make_migrations()
 
     print(time.ctime())
     print('Jumpserver version {}, more see https://www.jumpserver.org'.format(
         __version__))
     print('Quit the server with CONTROL-C.')
+
     threads = []
+    if 'gunicorn' in args.services:
+        threads.append(threading.Thread(target=start_gunicorn, args=()))
+    if 'celery' in args.services:
+        threads.append(threading.Thread(target=start_celery, args=()))
+    if 'beat' in args.services:
+        threads.append(threading.Thread(target=start_beat, args=()))
+    if 'all' in args.services:
+        _threads = []
         for func in (start_gunicorn, start_celery, start_beat):
-        t = Thread(target=func, args=())
+            t = threading.Thread(target=func, args=())
-        threads.append(t)
+            _threads.append(t)
-        t.start()
+        threads = _threads
 
     for t in threads:
-        t.join()
+        t.start()
+
+    if EXIT_EVENT.wait():
+        print("\n\n" + "####" * 30)
+        print("\n".join(EXIT_MSGS))
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser(description="Jumpserver start tools")
-
+    parser.add_argument("services", type=str, nargs='+', default="all",
-
+                        choices=("all", "gunicorn", "celery", "beat"),
-
+                        help="The service to start",
-
+                        )
+    args = parser.parse_args()
+    start_service(args.services)