mirror of https://github.com/Aidaho12/haproxy-wi
Pavel Loginov
4 years ago
8 changed files with 21 additions and 226 deletions
@ -1,79 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
import subprocess |
|
||||||
import time |
|
||||||
import argparse |
|
||||||
import os, sys |
|
||||||
sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) |
|
||||||
sys.path.append(os.path.join(sys.path[0], os.getcwd())) |
|
||||||
import funct |
|
||||||
import sql |
|
||||||
import signal |
|
||||||
|
|
||||||
class GracefulKiller: |
|
||||||
kill_now = False |
|
||||||
def __init__(self): |
|
||||||
signal.signal(signal.SIGINT, self.exit_gracefully) |
|
||||||
signal.signal(signal.SIGTERM, self.exit_gracefully) |
|
||||||
|
|
||||||
def exit_gracefully(self,signum, frame): |
|
||||||
self.kill_now = True |
|
||||||
|
|
||||||
def main(): |
|
||||||
servers = sql.select_alert() |
|
||||||
started_workers = get_worker() |
|
||||||
servers_list = [] |
|
||||||
|
|
||||||
for serv in servers: |
|
||||||
servers_list.append(serv[0]) |
|
||||||
|
|
||||||
need_kill=list(set(started_workers) - set(servers_list)) |
|
||||||
need_start=list(set(servers_list) - set(started_workers)) |
|
||||||
|
|
||||||
if need_kill: |
|
||||||
for serv in need_kill: |
|
||||||
kill_worker(serv) |
|
||||||
|
|
||||||
if need_start: |
|
||||||
for serv in need_start: |
|
||||||
start_worker(serv) |
|
||||||
|
|
||||||
def start_worker(serv): |
|
||||||
port = sql.get_setting('haproxy_sock_port') |
|
||||||
cmd = "tools/checker_worker.py %s --port %s &" % (serv, port) |
|
||||||
os.system(cmd) |
|
||||||
funct.logging("localhost", " Master started new worker for: "+serv, alerting=1) |
|
||||||
|
|
||||||
def kill_worker(serv): |
|
||||||
cmd = "ps ax |grep 'tools/checker_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv |
|
||||||
output, stderr = funct.subprocess_execute(cmd) |
|
||||||
funct.logging("localhost", " Master killed worker for: "+serv, alerting=1) |
|
||||||
if stderr: |
|
||||||
funct.logging("localhost", stderr, alerting=1) |
|
||||||
|
|
||||||
def kill_all_workers(): |
|
||||||
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill" |
|
||||||
output, stderr = funct.subprocess_execute(cmd) |
|
||||||
funct.logging("localhost", " Master killing all workers", alerting=1) |
|
||||||
if stderr: |
|
||||||
funct.logging("localhost", stderr, alerting=1) |
|
||||||
|
|
||||||
def get_worker(): |
|
||||||
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $7}'" |
|
||||||
output, stderr = funct.subprocess_execute(cmd) |
|
||||||
if stderr: |
|
||||||
funct.logging("localhost", stderr, alerting=1) |
|
||||||
return output |
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
funct.logging("localhost", " Checker master started", alerting=1) |
|
||||||
killer = GracefulKiller() |
|
||||||
|
|
||||||
while True: |
|
||||||
main() |
|
||||||
time.sleep(20) |
|
||||||
|
|
||||||
if killer.kill_now: |
|
||||||
break |
|
||||||
|
|
||||||
kill_all_workers() |
|
||||||
funct.logging("localhost", " Checker master shutdown", alerting=1) |
|
@ -1,106 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
import subprocess |
|
||||||
from subprocess import check_output, CalledProcessError |
|
||||||
import time |
|
||||||
import argparse |
|
||||||
import os |
|
||||||
import sys |
|
||||||
sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) |
|
||||||
sys.path.append(os.path.join(sys.path[0], os.getcwd())) |
|
||||||
import funct |
|
||||||
import signal |
|
||||||
|
|
||||||
class GracefulKiller: |
|
||||||
kill_now = False |
|
||||||
def __init__(self): |
|
||||||
signal.signal(signal.SIGINT, self.exit_gracefully) |
|
||||||
signal.signal(signal.SIGTERM, self.exit_gracefully) |
|
||||||
|
|
||||||
def exit_gracefully(self,signum, frame): |
|
||||||
self.kill_now = True |
|
||||||
|
|
||||||
def main(serv, port): |
|
||||||
port = str(port) |
|
||||||
firstrun = True |
|
||||||
currentstat = [] |
|
||||||
oldstat = [] |
|
||||||
readstats = "" |
|
||||||
killer = GracefulKiller() |
|
||||||
old_stat_service = "" |
|
||||||
|
|
||||||
while True: |
|
||||||
try: |
|
||||||
readstats = subprocess.check_output(["echo show stat | nc "+serv+" "+port], shell=True) |
|
||||||
except CalledProcessError as e: |
|
||||||
if firstrun == False: |
|
||||||
cur_stat_service = "error" |
|
||||||
if old_stat_service != cur_stat_service: |
|
||||||
alert = "Can't connect to HAProxy service at " + serv |
|
||||||
funct.telegram_send_mess(str(alert), ip=serv) |
|
||||||
funct.logging("localhost", " "+alert, alerting=1) |
|
||||||
|
|
||||||
firstrun = False |
|
||||||
old_stat_service = cur_stat_service |
|
||||||
time.sleep(60) |
|
||||||
continue |
|
||||||
except OSError as e: |
|
||||||
print(e) |
|
||||||
sys.exit() |
|
||||||
else: |
|
||||||
cur_stat_service = "Ok" |
|
||||||
if firstrun == False: |
|
||||||
if old_stat_service != cur_stat_service: |
|
||||||
alert = "Now UP HAProxy service at " + serv |
|
||||||
funct.telegram_send_mess(str(alert), ip=serv) |
|
||||||
funct.logging("localhost", " "+alert, alerting=1) |
|
||||||
firstrun = True |
|
||||||
time.sleep(5) |
|
||||||
old_stat_service = cur_stat_service |
|
||||||
|
|
||||||
vips = readstats.splitlines() |
|
||||||
|
|
||||||
for i in range(0,len(vips)): |
|
||||||
if "UP" in str(vips[i]): |
|
||||||
currentstat.append("UP") |
|
||||||
elif "DOWN" in str(vips[i]): |
|
||||||
currentstat.append("DOWN") |
|
||||||
elif "MAINT" in str(vips[i]): |
|
||||||
currentstat.append("MAINT") |
|
||||||
else: |
|
||||||
currentstat.append("none") |
|
||||||
|
|
||||||
if firstrun == False: |
|
||||||
if (currentstat[i] != oldstat[i] and currentstat[i]!="none") and ("FRONTEND" not in str(vips[i]) and "BACKEND" not in str(vips[i])): |
|
||||||
servername = str(vips[i]) |
|
||||||
servername = servername.split(",") |
|
||||||
realserver = servername[0] |
|
||||||
server = servername[1] |
|
||||||
alert = "Backend: "+realserver[2:]+", server: "+server+" has changed status to "+ currentstat[i] + " on " + serv + " HAProxy" |
|
||||||
funct.telegram_send_mess(str(alert), ip=serv) |
|
||||||
funct.logging("localhost", " "+alert, alerting=1) |
|
||||||
firstrun = False |
|
||||||
oldstat = currentstat |
|
||||||
currentstat = [] |
|
||||||
time.sleep(60) |
|
||||||
|
|
||||||
if killer.kill_now: |
|
||||||
break |
|
||||||
|
|
||||||
funct.logging("localhost", " Worker shutdown for: "+serv, alerting=1) |
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
parser = argparse.ArgumentParser(description='Check HAProxy servers state.', prog='check_haproxy.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
|
||||||
|
|
||||||
parser.add_argument('IP', help='Start check HAProxy server state at this ip', nargs='?', type=str) |
|
||||||
parser.add_argument('--port', help='Start check HAProxy server state at this port', nargs='?', default=1999, type=int) |
|
||||||
|
|
||||||
args = parser.parse_args() |
|
||||||
if args.IP is None: |
|
||||||
parser.print_help() |
|
||||||
import sys |
|
||||||
sys.exit() |
|
||||||
else: |
|
||||||
try: |
|
||||||
main(args.IP, args.port) |
|
||||||
except KeyboardInterrupt: |
|
||||||
pass |
|
@ -1,9 +0,0 @@ |
|||||||
/var/www/haproxy-wi/log/checker-error.log { |
|
||||||
daily |
|
||||||
rotate 10 |
|
||||||
missingok |
|
||||||
notifempty |
|
||||||
create 0644 apache apache |
|
||||||
dateext |
|
||||||
sharedscripts |
|
||||||
} |
|
@ -1,2 +0,0 @@ |
|||||||
if $programname startswith 'checker' then /var/www/haproxy-wi/log/checker-error.log |
|
||||||
& stop |
|
@ -1,19 +0,0 @@ |
|||||||
[Unit] |
|
||||||
Description=Haproxy backends state checker |
|
||||||
After=syslog.target network.target |
|
||||||
|
|
||||||
[Service] |
|
||||||
Type=simple |
|
||||||
WorkingDirectory=/var/www/haproxy-wi/app/ |
|
||||||
ExecStart=/var/www/haproxy-wi/app/tools/checker_master.py |
|
||||||
|
|
||||||
StandardOutput=syslog |
|
||||||
StandardError=syslog |
|
||||||
SyslogIdentifier=checker |
|
||||||
|
|
||||||
RestartSec=2s |
|
||||||
Restart=on-failure |
|
||||||
TimeoutStopSec=1s |
|
||||||
|
|
||||||
[Install] |
|
||||||
WantedBy=multi-user.target |
|
Loading…
Reference in new issue