diff --git a/app/create_db.py b/app/create_db.py index 194c962e..20441e4d 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -432,7 +432,7 @@ def update_db_v_42(**kwargs): def update_ver(**kwargs): con, cur = get_cur() - sql = """update version set version = '4.2.2.0'; """ + sql = """update version set version = '4.2.3.0'; """ try: cur.execute(sql) con.commit() diff --git a/app/funct.py b/app/funct.py index 5a2f2448..a71070a3 100644 --- a/app/funct.py +++ b/app/funct.py @@ -102,7 +102,13 @@ def telegram_send_mess(mess, **kwargs): for telegram in telegrams: token_bot = telegram[1] channel_name = telegram[2] - + + if token_bot == '' or channel_name == '': + mess = " Fatal: Can't send message. Add Telegram chanel before use alerting at this servers group" + print(mess) + logging('localhost', mess, haproxywi=1) + sys.exit() + if proxy is not None and proxy != '' and proxy != 'None': apihelper.proxy = {'https': proxy} try: @@ -280,26 +286,33 @@ def diff_config(oldcfg, cfg): pass -def get_sections(config): +def get_sections(config, **kwargs): record = False return_config = list() with open(config, 'r') as f: for line in f: - if ( - line.startswith('listen') or - line.startswith('frontend') or - line.startswith('backend') or - line.startswith('cache') or - line.startswith('defaults') or - line.startswith('global') or - line.startswith('#HideBlockEnd') or - line.startswith('#HideBlockStart') or - line.startswith('peers') or - line.startswith('resolvers') or - line.startswith('userlist') - ): + if kwargs.get('service') == 'nginx': + if 'server_name' in line: + line = line.split('server_name')[1] + line = line.split(';')[0] line = line.strip() return_config.append(line) + else: + if ( + line.startswith('listen') or + line.startswith('frontend') or + line.startswith('backend') or + line.startswith('cache') or + line.startswith('defaults') or + line.startswith('global') or + line.startswith('#HideBlockEnd') or + line.startswith('#HideBlockStart') or + line.startswith('peers') or + line.startswith('resolvers') or + line.startswith('userlist') + ): + line = line.strip() + return_config.append(line) return return_config @@ -834,22 +847,21 @@ def get_files(dir = get_config_var('configs', 'haproxy_save_configs_dir'), forma file = set() return_files = set() i = 0 - for files in glob.glob(os.path.join(dir,'*.'+format)): + for files in sorted(glob.glob(os.path.join(dir,'*.'+format))): if format == 'log': file += [(i, files.split('/')[5])] - else: + else: file.add(files.split('/')[-1]) i += 1 - files = sorted(file, reverse=True) - + files = file if format == 'cfg' or format == 'conf': - for file in files: + for file in files: ip = file.split("-") if serv == ip[0]: - return_files.add(file) + return_files.add(file) return sorted(return_files, reverse=True) else: - return files + return file def get_key(item): diff --git a/app/options.py b/app/options.py index 487af473..53edec01 100644 --- a/app/options.py +++ b/app/options.py @@ -136,26 +136,35 @@ if act == "overviewHapserverBackends": from jinja2 import Environment, FileSystemLoader env = Environment(loader=FileSystemLoader('templates/ajax'), autoescape=True) template = env.get_template('haproxyservers_backends.html') - - hap_configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir') + service = form.getvalue('service') + if service == 'haproxy': + configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir') + format = 'cfg' + elif service == 'nginx': + configs_dir = funct.get_config_var('configs', 'nginx_save_configs_dir') + format = 'conf' try: - sections = funct.get_sections(hap_configs_dir +funct.get_files()[0]) - except: + sections = funct.get_sections(configs_dir+funct.get_files(dir=configs_dir, format=format)[0], service=service) + except Exception as e: + funct.logging('localhost', str(e), haproxywi=1) try: - cfg = hap_configs_dir + serv + "-" + funct.get_data('config') + ".cfg" + cfg = configs_dir + serv + "-" + funct.get_data('config') + '.'+format except: funct.logging('localhost', ' Cannot generate cfg path', haproxywi=1) try: - error = funct.get_config(serv, cfg) + if service == 'nginx': + error = funct.get_config(serv, cfg, nginx=1) + else: + error = funct.get_config(serv, cfg) except: funct.logging('localhost', ' Cannot download config', haproxywi=1) try: - sections = funct.get_sections(cfg) + sections = funct.get_sections(cfg, service=service) except: funct.logging('localhost', ' Cannot get sections from config file', haproxywi=1) sections = 'Cannot get backends' - template = template.render(backends=sections, serv=serv) + template = template.render(backends=sections, serv=serv, service=service) print(template) @@ -264,20 +273,27 @@ if act == "overviewwaf": if act == "overviewServers": import asyncio - async def async_get_overviewServers(serv1, serv2): + async def async_get_overviewServers(serv1, serv2, service): server_status = () - commands = [ "top -u haproxy -b -n 1" ] - cmd = 'echo "show info" |nc %s %s -w 1|grep -e "Ver\|CurrConns\|Maxco\|MB\|Uptime:"' % (serv2, sql.get_setting('haproxy_sock_port')) - out = funct.subprocess_execute(cmd) - out1 = "" + if service == 'haproxy': + commands = [ "top -u haproxy -b -n 1" ] + else: + commands = [ "top -u nginx -b -n 1" ] - for k in out: - if "Ncat:" not in k: - for r in k: - out1 += r - out1 += "
" - else: - out1 = "Can\'t connect to HAproxy" + if service == 'haproxy': + cmd = 'echo "show info" |nc %s %s -w 1|grep -e "Ver\|CurrConns\|Maxco\|MB\|Uptime:"' % (serv2, sql.get_setting('haproxy_sock_port')) + out = funct.subprocess_execute(cmd) + out1 = "" + + for k in out: + if "Ncat:" not in k: + for r in k: + out1 += r + out1 += "
" + else: + out1 = "Can\'t connect to HAproxy" + else: + out1 = '' server_status = (serv1,serv2, out1, funct.ssh_command(serv2, commands)) return server_status @@ -292,19 +308,20 @@ if act == "overviewServers": cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE")) user_id = cookie.get('uuid') role = sql.get_user_role_by_uuid(user_id.value) - futures = [async_get_overviewServers(kwargs.get('server1'), kwargs.get('server2'))] + futures = [async_get_overviewServers(kwargs.get('server1'), kwargs.get('server2'), kwargs.get('service'))] for i, future in enumerate(asyncio.as_completed(futures)): result = await future servers.append(result) servers_sorted = sorted(servers, key=funct.get_key) - template = template.render(service_status=servers_sorted, role=role, id=kwargs.get('id')) + template = template.render(service_status=servers_sorted, role=role, id=kwargs.get('id'), service_page=service) print(template) id = form.getvalue('id') name = form.getvalue('name') + service = form.getvalue('service') ioloop = asyncio.get_event_loop() - ioloop.run_until_complete(get_runner_overviewServers(server1=name, server2=serv, id=id)) + ioloop.run_until_complete(get_runner_overviewServers(server1=name, server2=serv, id=id, service=service)) ioloop.close() diff --git a/app/overview.py b/app/overview.py index 5f17ff19..7386240d 100644 --- a/app/overview.py +++ b/app/overview.py @@ -33,6 +33,17 @@ try: grafana, stderr = funct.subprocess_execute(cmd) cmd = "ps ax |grep 'prometheus ' |grep -v grep|wc -l" prometheus, stderr = funct.subprocess_execute(cmd) + is_checker_workers = sql.select_alert() + i = 0 + for s in sql.select_alert(): + i += 1 + is_checker_worker = i + is_metrics_workers = sql.select_servers_metrics_for_master() + i = 0 + for s in is_metrics_workers: + i += 1 + is_metrics_worker = i + except: role = '' user = '' @@ -51,6 +62,9 @@ except: haproxy_wi_log = '' servers = '' stderr = '' + is_checker_worker = '' + is_metrics_worker = '' + token = '' template = template.render(h2 = 1, @@ -79,5 +93,7 @@ template = template.render(h2 = 1, versions = funct.versions(), haproxy_wi_log = funct.haproxy_wi_log(), servers = servers, + is_checker_worker = is_checker_worker, + is_metrics_worker = is_metrics_worker, token = token) print(template) \ No newline at end of file diff --git a/app/scripts/ansible/roles/grafana.yml b/app/scripts/ansible/roles/grafana.yml index 0d3509c9..2d80679c 100644 --- a/app/scripts/ansible/roles/grafana.yml +++ b/app/scripts/ansible/roles/grafana.yml @@ -32,4 +32,27 @@ - dashboard_id: 11879 revision_id: 3 datasource: prometheus - \ No newline at end of file + tasks: + - name: populate service facts + service_facts: + + - name: Open stat port for firewalld + firewalld: + port: "{{ item }}/tcp" + state: enabled + permanent: yes + immediate: yes + ignore_errors: yes + no_log: True + debugger: never + when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and ansible_facts.services["firewalld.service"]['state'] == "running" + with_items: [ "3000", "9090" ] + + - name: Open stat port for iptables + iptables: + chain: INPUT + destination_port: "{{ item }}" + jump: ACCEPT + protocol: tcp + ignore_errors: yes + with_items: [ "3000", "9090" ] \ No newline at end of file diff --git a/app/templates/ajax/haproxyservers_backends.html b/app/templates/ajax/haproxyservers_backends.html index ba97986b..5cd397d1 100644 --- a/app/templates/ajax/haproxyservers_backends.html +++ b/app/templates/ajax/haproxyservers_backends.html @@ -7,8 +7,12 @@ {{backends}} {% else %} {% for b in backends %} + {% if service == 'haproxy' %} {{b}} + {% else %} + {{b}} + {% endif %} {% endfor %} {% endif %} \ No newline at end of file diff --git a/app/templates/ajax/overviewServers.html b/app/templates/ajax/overviewServers.html index 875910d0..94299417 100644 --- a/app/templates/ajax/overviewServers.html +++ b/app/templates/ajax/overviewServers.html @@ -1,17 +1,27 @@ {%- for service in service_status -%}
+ {% if service_page == 'nginx' %} +
+ {% else %}
+ {% endif %} Server status - +
+ {% if service_page == 'haproxy' %}
{{ service.2 }}
+ {% endif %} + {% if service_page == 'nginx' %} +
+ {% else %}
+ {% endif %} 
 {{ service.3 }}
diff --git a/app/templates/hapservers.html b/app/templates/hapservers.html index c4333b07..bdd0b4cb 100644 --- a/app/templates/hapservers.html +++ b/app/templates/hapservers.html @@ -31,7 +31,7 @@ hostnamea.push(host) {% endfor %} -
+
{% for s in servers %} {% if serv %} @@ -45,35 +45,33 @@ getWafChartData('{{s.2}}') {% endfor %} {% endif %} - showOverviewServer('{{s.1}}', '{{s.2}}', '{{s.0}}'); + showOverviewServer('{{s.1}}', '{{s.2}}', '{{s.0}}', '{{service}}'); } showMetrics(); {% endif %} -
+ + {% if service == 'nginx' %} - {% if s.5.0.3 == 'active' %} {% else %} - {% endif %} - {{s.1}} + {% endif %} {% else %} - {% if s.5 != False %} {% else %} {% endif %} - {% if not serv %} - {{s.1}} + {% endif %} + {% if not serv %} + {{s.1}} {% else %} {{s.1}} {% endif %} - {% endif %} {% if service == 'haproxy' %} {% if s.6|int() >= 1 %}
{% if serv %} -
{% if service == 'haproxy' %} +
+ {% else %} +
+ {% endif %}
- Backends: + {% if service == 'haproxy' %} + Backends: + {% else %} + Virtual hosts: + {% endif %}
- {% endif %} {% endif %} -
{% endfor %}
{% if serv and service == 'haproxy' %} @@ -195,9 +198,9 @@

Are you sure?

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/app/templates/ovw.html b/app/templates/ovw.html index cd390267..63a32f08 100644 --- a/app/templates/ovw.html +++ b/app/templates/ovw.html @@ -111,7 +111,7 @@ {% if keep_alive|int() >= 1 %} {% else %} - + {% endif %} {% if role <= 1 %} @@ -127,7 +127,11 @@ {% if metrics_worker|int() >= 1 %} {% else %} - + {% if is_metrics_worker|int == 0 %} + + {% else %} + + {% endif %} {% endif %} {% if role <= 1 %} @@ -141,7 +145,11 @@ {% if checker_worker|int() >= 1 %} {% else %} + {% if is_checker_worker|int == 0 %} + + {% else %} + {% endif %} {% endif %} {% if role <= 1 %} diff --git a/app/tools/metrics_worker.py b/app/tools/metrics_worker.py index cfccec9a..b3754b94 100644 --- a/app/tools/metrics_worker.py +++ b/app/tools/metrics_worker.py @@ -33,14 +33,18 @@ def main(serv, port): except OSError as e: print(e) sys.exit() - readstats = readstats.decode(encoding='UTF-8') - metric = readstats.splitlines() - metrics = [] - for i in range(0,len(metric)): - metrics.append(metric[i]) + + try: + readstats = readstats.decode(encoding='UTF-8') + metric = readstats.splitlines() + metrics = [] + for i in range(0,len(metric)): + metrics.append(metric[i]) + + sql.insert_mentrics(serv, metrics[0], metrics[1], metrics[2], metrics[3]) + except Exception as e: + print(str(e)) - sql.insert_mentrics(serv, metrics[0], metrics[1], metrics[2], metrics[3]) - time.sleep(30) if killer.kill_now: diff --git a/inc/overview.js b/inc/overview.js index ed26ee22..356eba0e 100644 --- a/inc/overview.js +++ b/inc/overview.js @@ -41,12 +41,13 @@ function showHapserversCallBack(serv, hostnamea, service) { } } ); } -function overviewHapserverBackends(serv, hostnamea) { +function overviewHapserverBackends(serv, hostnamea, service) { $.ajax( { url: "options.py", data: { act: "overviewHapserverBackends", serv: serv[0], + service: service, token: $('#token').val() }, beforeSend: function() { @@ -85,7 +86,7 @@ function showOverviewCallBack(serv, hostnamea) { } } ); } -function showOverviewServer(name,ip,id) { +function showOverviewServer(name,ip,id, service) { $.ajax( { url: "options.py", data: { @@ -93,6 +94,7 @@ function showOverviewServer(name,ip,id) { name: name, serv: ip, id: id, + service: service, page: 'hapservers.py', token: $('#token').val() },