mirror of https://github.com/Aidaho12/haproxy-wi
Pavel Loginov
4 years ago
8 changed files with 21 additions and 226 deletions
@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env python3 |
||||
import subprocess |
||||
import time |
||||
import argparse |
||||
import os, sys |
||||
sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) |
||||
sys.path.append(os.path.join(sys.path[0], os.getcwd())) |
||||
import funct |
||||
import sql |
||||
import signal |
||||
|
||||
class GracefulKiller: |
||||
kill_now = False |
||||
def __init__(self): |
||||
signal.signal(signal.SIGINT, self.exit_gracefully) |
||||
signal.signal(signal.SIGTERM, self.exit_gracefully) |
||||
|
||||
def exit_gracefully(self,signum, frame): |
||||
self.kill_now = True |
||||
|
||||
def main(): |
||||
servers = sql.select_alert() |
||||
started_workers = get_worker() |
||||
servers_list = [] |
||||
|
||||
for serv in servers: |
||||
servers_list.append(serv[0]) |
||||
|
||||
need_kill=list(set(started_workers) - set(servers_list)) |
||||
need_start=list(set(servers_list) - set(started_workers)) |
||||
|
||||
if need_kill: |
||||
for serv in need_kill: |
||||
kill_worker(serv) |
||||
|
||||
if need_start: |
||||
for serv in need_start: |
||||
start_worker(serv) |
||||
|
||||
def start_worker(serv): |
||||
port = sql.get_setting('haproxy_sock_port') |
||||
cmd = "tools/checker_worker.py %s --port %s &" % (serv, port) |
||||
os.system(cmd) |
||||
funct.logging("localhost", " Master started new worker for: "+serv, alerting=1) |
||||
|
||||
def kill_worker(serv): |
||||
cmd = "ps ax |grep 'tools/checker_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv |
||||
output, stderr = funct.subprocess_execute(cmd) |
||||
funct.logging("localhost", " Master killed worker for: "+serv, alerting=1) |
||||
if stderr: |
||||
funct.logging("localhost", stderr, alerting=1) |
||||
|
||||
def kill_all_workers(): |
||||
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill" |
||||
output, stderr = funct.subprocess_execute(cmd) |
||||
funct.logging("localhost", " Master killing all workers", alerting=1) |
||||
if stderr: |
||||
funct.logging("localhost", stderr, alerting=1) |
||||
|
||||
def get_worker(): |
||||
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $7}'" |
||||
output, stderr = funct.subprocess_execute(cmd) |
||||
if stderr: |
||||
funct.logging("localhost", stderr, alerting=1) |
||||
return output |
||||
|
||||
if __name__ == "__main__": |
||||
funct.logging("localhost", " Checker master started", alerting=1) |
||||
killer = GracefulKiller() |
||||
|
||||
while True: |
||||
main() |
||||
time.sleep(20) |
||||
|
||||
if killer.kill_now: |
||||
break |
||||
|
||||
kill_all_workers() |
||||
funct.logging("localhost", " Checker master shutdown", alerting=1) |
||||
@ -1,106 +0,0 @@
|
||||
#!/usr/bin/env python3 |
||||
import subprocess |
||||
from subprocess import check_output, CalledProcessError |
||||
import time |
||||
import argparse |
||||
import os |
||||
import sys |
||||
sys.path.append(os.path.join(sys.path[0], os.path.dirname(os.getcwd()))) |
||||
sys.path.append(os.path.join(sys.path[0], os.getcwd())) |
||||
import funct |
||||
import signal |
||||
|
||||
class GracefulKiller: |
||||
kill_now = False |
||||
def __init__(self): |
||||
signal.signal(signal.SIGINT, self.exit_gracefully) |
||||
signal.signal(signal.SIGTERM, self.exit_gracefully) |
||||
|
||||
def exit_gracefully(self,signum, frame): |
||||
self.kill_now = True |
||||
|
||||
def main(serv, port): |
||||
port = str(port) |
||||
firstrun = True |
||||
currentstat = [] |
||||
oldstat = [] |
||||
readstats = "" |
||||
killer = GracefulKiller() |
||||
old_stat_service = "" |
||||
|
||||
while True: |
||||
try: |
||||
readstats = subprocess.check_output(["echo show stat | nc "+serv+" "+port], shell=True) |
||||
except CalledProcessError as e: |
||||
if firstrun == False: |
||||
cur_stat_service = "error" |
||||
if old_stat_service != cur_stat_service: |
||||
alert = "Can't connect to HAProxy service at " + serv |
||||
funct.telegram_send_mess(str(alert), ip=serv) |
||||
funct.logging("localhost", " "+alert, alerting=1) |
||||
|
||||
firstrun = False |
||||
old_stat_service = cur_stat_service |
||||
time.sleep(60) |
||||
continue |
||||
except OSError as e: |
||||
print(e) |
||||
sys.exit() |
||||
else: |
||||
cur_stat_service = "Ok" |
||||
if firstrun == False: |
||||
if old_stat_service != cur_stat_service: |
||||
alert = "Now UP HAProxy service at " + serv |
||||
funct.telegram_send_mess(str(alert), ip=serv) |
||||
funct.logging("localhost", " "+alert, alerting=1) |
||||
firstrun = True |
||||
time.sleep(5) |
||||
old_stat_service = cur_stat_service |
||||
|
||||
vips = readstats.splitlines() |
||||
|
||||
for i in range(0,len(vips)): |
||||
if "UP" in str(vips[i]): |
||||
currentstat.append("UP") |
||||
elif "DOWN" in str(vips[i]): |
||||
currentstat.append("DOWN") |
||||
elif "MAINT" in str(vips[i]): |
||||
currentstat.append("MAINT") |
||||
else: |
||||
currentstat.append("none") |
||||
|
||||
if firstrun == False: |
||||
if (currentstat[i] != oldstat[i] and currentstat[i]!="none") and ("FRONTEND" not in str(vips[i]) and "BACKEND" not in str(vips[i])): |
||||
servername = str(vips[i]) |
||||
servername = servername.split(",") |
||||
realserver = servername[0] |
||||
server = servername[1] |
||||
alert = "Backend: "+realserver[2:]+", server: "+server+" has changed status to "+ currentstat[i] + " on " + serv + " HAProxy" |
||||
funct.telegram_send_mess(str(alert), ip=serv) |
||||
funct.logging("localhost", " "+alert, alerting=1) |
||||
firstrun = False |
||||
oldstat = currentstat |
||||
currentstat = [] |
||||
time.sleep(60) |
||||
|
||||
if killer.kill_now: |
||||
break |
||||
|
||||
funct.logging("localhost", " Worker shutdown for: "+serv, alerting=1) |
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description='Check HAProxy servers state.', prog='check_haproxy.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
||||
|
||||
parser.add_argument('IP', help='Start check HAProxy server state at this ip', nargs='?', type=str) |
||||
parser.add_argument('--port', help='Start check HAProxy server state at this port', nargs='?', default=1999, type=int) |
||||
|
||||
args = parser.parse_args() |
||||
if args.IP is None: |
||||
parser.print_help() |
||||
import sys |
||||
sys.exit() |
||||
else: |
||||
try: |
||||
main(args.IP, args.port) |
||||
except KeyboardInterrupt: |
||||
pass |
||||
@ -1,9 +0,0 @@
|
||||
/var/www/haproxy-wi/log/checker-error.log { |
||||
daily |
||||
rotate 10 |
||||
missingok |
||||
notifempty |
||||
create 0644 apache apache |
||||
dateext |
||||
sharedscripts |
||||
} |
||||
@ -1,2 +0,0 @@
|
||||
if $programname startswith 'checker' then /var/www/haproxy-wi/log/checker-error.log |
||||
& stop |
||||
@ -1,19 +0,0 @@
|
||||
[Unit] |
||||
Description=Haproxy backends state checker |
||||
After=syslog.target network.target |
||||
|
||||
[Service] |
||||
Type=simple |
||||
WorkingDirectory=/var/www/haproxy-wi/app/ |
||||
ExecStart=/var/www/haproxy-wi/app/tools/checker_master.py |
||||
|
||||
StandardOutput=syslog |
||||
StandardError=syslog |
||||
SyslogIdentifier=checker |
||||
|
||||
RestartSec=2s |
||||
Restart=on-failure |
||||
TimeoutStopSec=1s |
||||
|
||||
[Install] |
||||
WantedBy=multi-user.target |
||||
Loading…
Reference in new issue