diff --git a/apps/__init__.py b/apps/__init__.py index 5c66b357e..6110e1346 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -2,4 +2,4 @@ # -*- coding: utf-8 -*- # -__version__ = "0.5.0" +__version__ = "1.0.0" diff --git a/jms b/jms new file mode 100755 index 000000000..0613e6287 --- /dev/null +++ b/jms @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import subprocess +import threading +import time +import argparse +import sys +import signal + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(BASE_DIR) + +from apps import __version__ + +try: + from config import config as CONFIG +except ImportError: + print("Could not find config file, `cp config_example.py config.py`") + sys.exit(1) + +os.environ["PYTHONIOENCODING"] = "UTF-8" +APPS_DIR = os.path.join(BASE_DIR, 'apps') +LOG_DIR = os.path.join(BASE_DIR, 'logs') +TMP_DIR = os.path.join(BASE_DIR, 'tmp') +HTTP_HOST = CONFIG.HTTP_BIND_HOST or '127.0.0.1' +HTTP_PORT = CONFIG.HTTP_LISTEN_PORT or 8080 +DEBUG = CONFIG.DEBUG +LOG_LEVEL = CONFIG.LOG_LEVEL + +WORKERS = 4 +DAEMON = False + +EXIT_EVENT = threading.Event() +all_services = ['gunicorn', 'celery', 'beat'] + +try: + os.makedirs(os.path.join(BASE_DIR, "data", "static")) + os.makedirs(os.path.join(BASE_DIR, "data", "media")) +except: + pass + + +def make_migrations(): + print("Check database structure change ...") + os.chdir(os.path.join(BASE_DIR, 'apps')) + subprocess.call('python3 manage.py migrate', shell=True) + + +def collect_static(): + print("Collect static files") + os.chdir(os.path.join(BASE_DIR, 'apps')) + subprocess.call('python3 manage.py collectstatic --no-input', shell=True) + + +def prepare(): + make_migrations() + collect_static() + + +def check_pid(pid): + """ Check For the existence of a unix pid. """ + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + +def get_pid_file_path(service): + return os.path.join(TMP_DIR, '{}.pid'.format(service)) + + +def get_log_file_path(service): + return os.path.join(LOG_DIR, '{}.log'.format(service)) + + +def get_pid(service): + pid_file = get_pid_file_path(service) + if os.path.isfile(pid_file): + with open(pid_file) as f: + return int(f.read().strip()) + return 0 + + +def is_running(s, unlink=True): + pid_file = get_pid_file_path(s) + + if os.path.isfile(pid_file): + with open(pid_file, 'r') as f: + pid = get_pid(s) + if check_pid(pid): + return True + + if unlink: + os.unlink(pid_file) + return False + + +def parse_service(s): + if s == 'all': + return all_services + else: + return [s] + + +def start_gunicorn(): + print("\n- Start Gunicorn WSGI HTTP Server") + prepare() + service = 'gunicorn' + bind = '{}:{}'.format(HTTP_HOST, HTTP_PORT) + log_format = '%(h)s %(t)s "%(r)s" %(s)s %(b)s ' + pid_file = get_pid_file_path(service) + log_file = get_log_file_path(service) + + cmd = [ + 'gunicorn', 'jumpserver.wsgi', + '-b', bind, + '-w', str(WORKERS), + '--access-logformat', log_format, + '-p', pid_file, + ] + + if DAEMON: + cmd.extend([ + '--access-logfile', log_file, + '--daemon', + ]) + else: + cmd.extend([ + '--access-logfile', '-' + ]) + if DEBUG: + cmd.append('--reload') + p = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr, cwd=APPS_DIR) + return p + + +def start_celery(): + print("\n- Start Celery as Distributed Task Queue") + # Todo: Must set this environment, otherwise not no ansible result return + os.environ.setdefault('PYTHONOPTIMIZE', '1') + + if os.getuid() == 0: + os.environ.setdefault('C_FORCE_ROOT', '1') + + service = 'celery' + pid_file = get_pid_file_path(service) + + cmd = [ + 'celery', 'worker', + '-A', 'common', + '-l', LOG_LEVEL.lower(), + '--pidfile', pid_file, + '-c', str(WORKERS), + ] + if DAEMON: + cmd.extend([ + '--logfile', os.path.join(LOG_DIR, 'celery.log'), + '--detach', + ]) + p = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr, cwd=APPS_DIR) + return p + + +def start_beat(): + print("\n- Start Beat as Periodic Task Scheduler") + + pid_file = get_pid_file_path('beat') + log_file = get_log_file_path('beat') + + os.environ.setdefault('PYTHONOPTIMIZE', '1') + if os.getuid() == 0: + os.environ.setdefault('C_FORCE_ROOT', '1') + + scheduler = "django_celery_beat.schedulers:DatabaseScheduler" + cmd = [ + 'celery', 'beat', + '-A', 'common', + '--pidfile', pid_file, + '-l', LOG_LEVEL, + '--scheduler', scheduler, + '--max-interval', '60' + ] + if DAEMON: + cmd.extend([ + '--logfile', log_file, + '--detach', + ]) + p = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr, cwd=APPS_DIR) + return p + + +def start_service(s): + print(time.ctime()) + print('Jumpserver version {}, more see https://www.jumpserver.org'.format( + __version__)) + + services_handler = { + "gunicorn": start_gunicorn, + "celery": start_celery, + "beat": start_beat + } + + services_set = parse_service(s) + processes = [] + for i in services_set: + if is_running(i): + show_service_status(i) + continue + func = services_handler.get(i) + p = func() + processes.append(p) + + time.sleep(5) + for i in services_set: + if not is_running(i): + print("Error: {} start error".format(i)) + stop_multi_services(services_set) + + stop_event = threading.Event() + + if not DAEMON: + signal.signal(signal.SIGTERM, lambda x, y: stop_event.set()) + while not stop_event.is_set(): + try: + time.sleep(10) + except KeyboardInterrupt: + stop_event.set() + break + + print("Stop services") + for p in processes: + p.terminate() + + for i in services_set: + stop_service(i) + else: + print() + show_service_status(s) + + +def stop_service(s, sig=15): + services_set = parse_service(s) + for s in services_set: + if not is_running(s): + show_service_status(s) + continue + print("Stop service: {}".format(s)) + pid = get_pid(s) + os.kill(pid, sig) + + +def stop_multi_services(services): + for s in services: + stop_service(s, sig=9) + + +def stop_service_force(s): + stop_service(s, sig=9) + + +def show_service_status(s): + services_set = parse_service(s) + for ns in services_set: + if is_running(ns): + pid = get_pid(ns) + print("{} is running: {}".format(ns, pid)) + else: + print("{} is stopped".format(ns)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=""" + Jumpserver service control tools; + + Example: \r\n + + %(prog)s start all -d; + """ + ) + parser.add_argument( + 'action', type=str, + choices=("start", "stop", "restart", "status"), + help="Action to run" + ) + parser.add_argument( + "service", type=str, default="all", nargs="?", + choices=("all", "gunicorn", "celery", "beat"), + help="The service to start", + ) + parser.add_argument('-d', '--daemon', nargs="?", const=1) + parser.add_argument('-w', '--worker', type=int, nargs="?", const=4) + args = parser.parse_args() + if args.daemon: + DAEMON = True + + if args.worker: + WORKERS = args.worker + + action = args.action + srv = args.service + + if action == "start": + start_service(srv) + elif action == "stop": + stop_service(srv) + elif action == "restart": + DAEMON = True + stop_service(srv) + time.sleep(5) + start_service(srv) + else: + show_service_status(srv) diff --git a/run_server.py b/run_server.py index 70871582a..c7ec7bccb 100644 --- a/run_server.py +++ b/run_server.py @@ -1,158 +1,11 @@ #!/usr/bin/env python3 # coding: utf-8 -import os -import subprocess -import threading -import time -import argparse import sys - -from apps import __version__ - -try: - from config import config as CONFIG -except ImportError: - CONFIG = type('_', (), {'__getattr__': lambda *arg: None})() - -os.environ["PYTHONIOENCODING"] = "UTF-8" - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -APPS_DIR = os.path.join(BASE_DIR, 'apps') -HTTP_HOST = CONFIG.HTTP_BIND_HOST or '127.0.0.1' -HTTP_PORT = CONFIG.HTTP_LISTEN_PORT or 8080 -DEBUG = CONFIG.DEBUG -LOG_LEVEL = CONFIG.LOG_LEVEL -WORKERS = 4 - -EXIT_EVENT = threading.Event() -processes = {} - -try: - os.makedirs(os.path.join(BASE_DIR, "data", "static")) - os.makedirs(os.path.join(BASE_DIR, "data", "media")) -except: - pass - - -def make_migrations(): - print("Check database change, make migrations") - os.chdir(os.path.join(BASE_DIR, 'apps')) - subprocess.call('python3 manage.py migrate', shell=True) - - -def collect_static(): - print("Collect static files") - os.chdir(os.path.join(BASE_DIR, 'apps')) - subprocess.call('python3 manage.py collectstatic --no-input', shell=True) - - -def start_gunicorn(): - print("- Start Gunicorn WSGI HTTP Server") - make_migrations() - collect_static() - os.chdir(APPS_DIR) - cmd = "gunicorn jumpserver.wsgi -b {}:{} -w {} ".format( - HTTP_HOST, HTTP_PORT, WORKERS - ) - log_format = '%(h)s %(t)s "%(r)s" %(s)s %(b)s ' - log = " --access-logfile - --access-logformat '{}' ".format(log_format) - cmd += log - if DEBUG: - cmd += " --reload" - p = subprocess.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr) - return p - - -def start_celery(): - print("- Start Celery as Distributed Task Queue") - os.chdir(APPS_DIR) - # Todo: Must set this environment, otherwise not no ansible result return - os.environ.setdefault('PYTHONOPTIMIZE', '1') - - cmd = """ - export C_FORCE_ROOT=1; - celery -A common worker -l {} - """.format(LOG_LEVEL.lower()) - - p = subprocess.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr) - return p - - -def start_beat(): - print("- Start Beat as Periodic Task Scheduler") - os.chdir(APPS_DIR) - os.environ.setdefault('PYTHONOPTIMIZE', '1') - os.environ.setdefault('C_FORCE_ROOT', '1') - pidfile = '/tmp/beat.pid' - if os.path.exists(pidfile): - print("Beat pid file `{}` exist, remove it".format(pidfile)) - os.unlink(pidfile) - time.sleep(0.5) - - if os.path.exists(pidfile): - print("Beat pid file `{}` exist yet, may be something wrong".format(pidfile)) - os.unlink(pidfile) - time.sleep(0.5) - - scheduler = "django_celery_beat.schedulers:DatabaseScheduler" - options = "--pidfile {} -l {} --scheduler {} --max-interval 60".format( - pidfile, LOG_LEVEL, scheduler, - ) - cmd = 'celery -A common beat {} '.format(options) - p = subprocess.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr) - return p - - -def start_service(services): - print(time.ctime()) - print('Jumpserver version {}, more see https://www.jumpserver.org'.format( - __version__)) - print('Quit the server with CONTROL-C.') - - services_all = { - "gunicorn": start_gunicorn, - "celery": start_celery, - "beat": start_beat - } - - if 'all' in services: - for name, func in services_all.items(): - processes[name] = func() - else: - for name in services: - func = services_all.get(name) - processes[name] = func() - - stop_event = threading.Event() - while not stop_event.is_set(): - for name, proc in processes.items(): - if proc.poll() is not None: - print("\n\n" + "####"*10 + " ERROR OCCUR " + "####"*10) - print("Start service {} [FAILED]".format(name)) - for _, p in processes.items(): - p.terminate() - stop_event.set() - print("Exited".format(name)) - break - time.sleep(5) - - -def stop_service(): - for name, proc in processes.items(): - print("Stop service {}".format(name)) - proc.terminate() - - if os.path.exists("/tmp/beat.pid"): - os.unlink('/tmp/beat.pid') +import subprocess if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Jumpserver start tools") - parser.add_argument("services", type=str, nargs='+', default="all", - choices=("all", "gunicorn", "celery", "beat"), - help="The service to start", - ) - args = parser.parse_args() - start_service(args.services) + subprocess.call('python3 jms start all', shell=True, + stdin=sys.stdin, stdout=sys.stdout) diff --git a/supervisord.conf b/supervisord.conf deleted file mode 100644 index a7f9540da..000000000 --- a/supervisord.conf +++ /dev/null @@ -1,143 +0,0 @@ -; Notes: -; - Shell expansion ("~" or "$HOME") is not supported. Environment -; variables can be expanded using this syntax: "%(ENV_HOME)s". -; - Quotes around values are not supported, except in the case of -; the environment= options as shown below. -; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". -; - Command will be truncated if it looks like a config file comment, e.g. -; "command=bash -c 'foo ; bar'" will truncate to "command=bash -c 'foo ". - -[unix_http_server] -file=/tmp/supervisor.sock ; the path to the socket file -;chmod=0700 ; socket file mode (default 0700) -;chown=nobody:nogroup ; socket file uid:gid owner -;username=user ; default is no username (open server) -;password=123 ; default is no password (open server) - -;[inet_http_server] ; inet (TCP) server disabled by default -;port=127.0.0.1:9001 ; ip_address:port specifier, *:port for all iface -;username=user ; default is no username (open server) -;password=123 ; default is no password (open server) - -[supervisord] -logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log -logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB -logfile_backups=10 ; # of main logfile backups; 0 means none, default 10 -loglevel=info ; log level; default info; others: debug,warn,trace -pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid -nodaemon=true; start in foreground if true; default false -minfds=1024 ; min. avail startup file descriptors; default 1024 -minprocs=200 ; min. avail process descriptors;default 200 -;umask=022 ; process file creation umask; default 022 -;user=chrism ; default is current user, required if root -;identifier=supervisor ; supervisord identifier, default is 'supervisor' -;directory=/tmp ; default is not to cd during start -;nocleanup=true ; don't clean up tempfiles at start; default false -;childlogdir=/tmp ; 'AUTO' child log dir, default $TEMP -;environment=KEY="value" ; key value pairs to add to environment -;strip_ansi=false ; strip ansi escape codes in logs; def. false - -; The rpcinterface:supervisor section must remain in the config file for -; RPC (supervisorctl/web interface) to work. Additional interfaces may be -; added by defining them in separate [rpcinterface:x] sections. - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -; The supervisorctl section configures how supervisorctl will connect to -; supervisord. configure it match the settings in either the unix_http_server -; or inet_http_server section. - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket -;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket -;username=chris ; should be same as in [*_http_server] if set -;password=123 ; should be same as in [*_http_server] if set -;prompt=mysupervisor ; cmd line prompt (default "supervisor") -;history_file=~/.sc_history ; use readline history if available - -; The sample program section below shows all possible program subsection values. -; Create one or more 'real' program: sections to be able to control them under -; supervisor. - -[program:jumpserver] -command=/bin/cat ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=999 ; the relative start priority (default 999) -;autostart=true ; start at supervisord start (default: true) -;startsecs=1 ; # of secs prog must stay up to be running (def. 1) -;startretries=3 ; max # of serial start failures when starting (default 3) -;autorestart=unexpected ; when to restart if exited after running (def: unexpected) -;exitcodes=0,2 ; 'expected' exit codes used with autorestart (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;stopasgroup=false ; send stop signal to the UNIX process group (default false) -;killasgroup=false ; SIGKILL the UNIX process group (def false) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=true ; redirect proc stderr to stdout (default false) -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) -;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) -;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A="1",B="2" ; process environment additions (def no adds) -;serverurl=AUTO ; override serverurl computation (childutils) - -; The sample eventlistener section below shows all possible eventlistener -; subsection values. Create one or more 'real' eventlistener: sections to be -; able to handle event notifications sent by supervisord. - -;[eventlistener:theeventlistenername] -;command=/bin/eventlistener ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;events=EVENT ; event notif. types to subscribe to (req'd) -;buffer_size=10 ; event buffer queue size (default 10) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=-1 ; the relative start priority (default -1) -;autostart=true ; start at supervisord start (default: true) -;startsecs=1 ; # of secs prog must stay up to be running (def. 1) -;startretries=3 ; max # of serial start failures when starting (default 3) -;autorestart=unexpected ; autorestart if exited after running (def: unexpected) -;exitcodes=0,2 ; 'expected' exit codes used with autorestart (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;stopasgroup=false ; send stop signal to the UNIX process group (default false) -;killasgroup=false ; SIGKILL the UNIX process group (def false) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=false ; redirect_stderr=true is not allowed for eventlisteners -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A="1",B="2" ; process environment additions -;serverurl=AUTO ; override serverurl computation (childutils) - -; The sample group section below shows all possible group values. Create one -; or more 'real' group: sections to create "heterogeneous" process groups. - -;[group:thegroupname] -;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions -;priority=999 ; the relative start priority (default 999) - -; The [include] section can just contain the "files" setting. This -; setting can list multiple files (separated by whitespace or -; newlines). It can also contain wildcards. The filenames are -; interpreted as relative to this file. Included files *cannot* -; include files themselves. - -;[include] -;files = relative/directory/*.ini