From b66c4d079330ea2f7c9c6da42099e88126f1bd31 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Fri, 3 May 2024 15:22:58 +0300 Subject: [PATCH] v7.2.5.0 https://roxy-wi.org/changelog#7_2_5 --- app/create_db.py | 2 +- app/modules/db/metric.py | 4 + app/modules/roxywi/auth.py | 2 +- app/modules/roxywi/metrics.py | 17 +- app/modules/server/ssh.py | 40 ++++- app/routes/admin/routes.py | 39 ++++- app/routes/main/routes.py | 35 ---- app/routes/metric/routes.py | 40 ++++- app/static/css/style-6.3.9.css | 12 +- app/static/js/metrics.js | 177 +++++++++++-------- app/static/js/users.js | 33 ++-- app/templates/admin.html | 34 ++-- app/templates/include/admin_servers.html | 8 +- app/templates/include/admin_settings.html | 6 +- app/templates/include/admin_ssh.html | 4 +- app/templates/include/admin_users.html | 10 +- app/templates/include/admins_dialogs.html | 6 +- app/templates/include/main_menu.html | 22 +-- app/templates/metrics.html | 3 - app/templates/servers.html | 80 --------- app/templates/service.html | 10 +- app/templates/waf.html | 3 - inc/script.js | 197 +++++++++------------- 23 files changed, 390 insertions(+), 394 deletions(-) delete mode 100644 app/templates/servers.html diff --git a/app/create_db.py b/app/create_db.py index f8316d02..fac37946 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -755,7 +755,7 @@ def update_db_v_7_2_3(): def update_ver(): try: - Version.update(version='7.2.4.0').execute() + Version.update(version='7.2.5.0').execute() except Exception: print('Cannot update version') diff --git a/app/modules/db/metric.py b/app/modules/db/metric.py index 477d171c..ed34e390 100644 --- a/app/modules/db/metric.py +++ b/app/modules/db/metric.py @@ -174,6 +174,8 @@ def select_metrics(serv, service, **kwargs): date_from = "and date > CONVERT_TZ(NOW(),'SYSTEM','+0:00') - INTERVAL 360 minute group by `date` div 300" elif kwargs.get('time_range') == '720': date_from = "and date > CONVERT_TZ(NOW(),'SYSTEM','+0:00') - INTERVAL 720 minute group by `date` div 500" + elif kwargs.get('time_range') == '1': + date_from = "and date > CONVERT_TZ(NOW(),'SYSTEM','+0:00') - INTERVAL 1 minute" else: date_from = "and date > CONVERT_TZ(NOW(),'SYSTEM','+0:00') - INTERVAL 30 minute" sql = """ select * from {metrics_table} where serv = '{serv}' {date_from} order by `date` asc """.format( @@ -188,6 +190,8 @@ def select_metrics(serv, service, **kwargs): date_from = "and date > datetime('now', '-360 minutes', 'UTC') and rowid % 7 = 0" elif kwargs.get('time_range') == '720': date_from = "and date > datetime('now', '-720 minutes', 'UTC') and rowid % 9 = 0" + elif kwargs.get('time_range') == '1': + date_from = "and date > datetime('now', '-1 minutes', 'UTC')" else: date_from = "and date > datetime('now', '-30 minutes', 'UTC')" diff --git a/app/modules/roxywi/auth.py b/app/modules/roxywi/auth.py index dfb6bbc2..ca91f063 100644 --- a/app/modules/roxywi/auth.py +++ b/app/modules/roxywi/auth.py @@ -61,7 +61,7 @@ def is_admin(level=1, **kwargs): def page_for_admin(level=1) -> None: if not is_admin(level=level): - return abort(400, 'bad permission') + return abort(404, 'Not found') def check_in_ldap(user, password): diff --git a/app/modules/roxywi/metrics.py b/app/modules/roxywi/metrics.py index 0b2a9068..08e29d46 100644 --- a/app/modules/roxywi/metrics.py +++ b/app/modules/roxywi/metrics.py @@ -11,14 +11,14 @@ def show_ram_metrics(metrics_type: str) -> dict: if metrics_type == '1': rams_list = psutil.virtual_memory() - rams += str(round(rams_list.total / 1048576, 2)) + ' ' rams += str(round(rams_list.used / 1048576, 2)) + ' ' rams += str(round(rams_list.free / 1048576, 2)) + ' ' rams += str(round(rams_list.shared / 1048576, 2)) + ' ' rams += str(round(rams_list.cached / 1048576, 2)) + ' ' rams += str(round(rams_list.available / 1048576, 2)) + ' ' + rams += str(round(rams_list.total / 1048576, 2)) + ' ' else: - commands = ["free -m |grep Mem |awk '{print $2,$3,$4,$5,$6,$7}'"] + commands = ["free -m |grep Mem |awk '{print $3,$4,$5,$6,$7,$2}'"] metric, error = server_mod.subprocess_execute(commands[0]) for i in metric: @@ -34,7 +34,8 @@ def show_cpu_metrics(metrics_type: str) -> dict: cpus = '' if metrics_type == '1': - cpus_list = psutil.cpu_times_percent(interval=1, percpu=False) + total = psutil.cpu_percent(0.5) + cpus_list = psutil.cpu_times_percent(interval=0.5, percpu=False) cpus += str(cpus_list.user) + ' ' cpus += str(cpus_list.system) + ' ' cpus += str(cpus_list.nice) + ' ' @@ -43,13 +44,15 @@ def show_cpu_metrics(metrics_type: str) -> dict: cpus += str(cpus_list.irq) + ' ' cpus += str(cpus_list.softirq) + ' ' cpus += str(cpus_list.steal) + ' ' + cpus += str(total) + ' ' else: - commands = [ - "top -b -n 1 |grep Cpu |awk -F':' '{print $2}'|awk -F' ' 'BEGIN{ORS=\" \";} { for (i=1;i<=NF;i+=2) print $i}'"] - metric, error = server_mod.subprocess_execute(commands[0]) - + cmd = "top -d 0.5 -b -n2 | grep 'Cpu(s)'|tail -n 1 | awk '{print $2 + $4}'" + total, error = server_mod.subprocess_execute(cmd) + cmd = "top -b -n 1 |grep Cpu |awk -F':' '{print $2}'|awk -F' ' 'BEGIN{ORS=\" \";} { for (i=1;i<=NF;i+=2) print $i}'" + metric, error = server_mod.subprocess_execute(cmd) for i in metric: cpus = i + cpus += f'{total[0]}' metrics['chartData']['cpus'] = cpus diff --git a/app/modules/server/ssh.py b/app/modules/server/ssh.py index 14db2cc6..6744222e 100644 --- a/app/modules/server/ssh.py +++ b/app/modules/server/ssh.py @@ -2,6 +2,7 @@ import os from cryptography.fernet import Fernet import paramiko +from paramiko import RSAKey, DSSKey, ECDSAKey, Ed25519Key, PKey from flask import render_template, request import app.modules.db.cred as cred_sql @@ -117,7 +118,36 @@ def create_ssh_cread_api(name: str, enable: str, group: str, username: str, pass roxywi_common.handle_exceptions(e, 'Roxy-WI server', f'Cannot create SSH credentials {name}', roxywi=1) +def get_key_class_name(uploaded_key, passphrase): + for pkey_class in (ECDSAKey, RSAKey, DSSKey, Ed25519Key): + try: + key = pkey_class.from_private_key(uploaded_key, passphrase) + class_name = str(pkey_class).split('.')[-1].rstrip(">'") + print(class_name) + + # return class_name + return key + + except Exception as e: + print("An exception occurred: {}".format(e)) + pass + + +def proper_method_call(filepath, passphrase): + key_class_name = get_key_class_name(filepath, passphrase) + print('key_class_name',key_class_name) + key_class = getattr(paramiko, key_class_name) + key_method = getattr(key_class, "from_private_key") + print('filepath',filepath) + try: + key = key_method(filepath, passphrase) + except Exception as e: + raise Exception(f'something went wrong: {e}') + return key + + def upload_ssh_key(name: str, user_group: str, key: str, passphrase: str) -> str: + import io if '..' in name: raise Exception('error: nice try') @@ -125,9 +155,15 @@ def upload_ssh_key(name: str, user_group: str, key: str, passphrase: str) -> str raise Exception('error: please select credentials first') try: - key = paramiko.pkey.load_private_key(key, password=passphrase) + print('key1',key) + key = io.StringIO(key) + print('key2',key) + # key = paramiko.pkey.load_private_key(key, password=passphrase) + key = paramiko.Ed25519Key.from_private_key(key, password=passphrase) + # key = get_key_class_name(key, passphrase) + # key = proper_method_call(key, passphrase) except Exception as e: - raise Exception(f'error: Cannot save SSH key file: {e}') + raise Exception(f'error: Cannot read SSH key: {e}') lib_path = get_config.get_config_var('main', 'lib_path') full_dir = f'{lib_path}/keys/' diff --git a/app/routes/admin/routes.py b/app/routes/admin/routes.py index c1ade31b..51fea83e 100644 --- a/app/routes/admin/routes.py +++ b/app/routes/admin/routes.py @@ -33,22 +33,35 @@ def before_request(): @bp.route('') @get_user_params() def admin(): - roxywi_auth.page_for_admin() + roxywi_auth.page_for_admin(level=2) + user_group = roxywi_common.get_user_group(id=1) + if g.user_params['role'] == 1: + users = user_sql.select_users() + servers = server_sql.select_servers(full=1) + masters = server_sql.select_servers(get_master_servers=1) + sshs = cred_sql.select_ssh() + else: + users = user_sql.select_users(group=user_group) + servers = roxywi_common.get_dick_permit(virt=1, disable=0, only_group=1) + masters = server_sql.select_servers(get_master_servers=1, uuid=g.user_params['user_uuid']) + sshs = cred_sql.select_ssh(group=user_group) + + kwargs = { 'lang': g.user_params['lang'], - 'users': user_sql.select_users(), + 'users': users, 'groups': group_sql.select_groups(), - 'sshs': cred_sql.select_ssh(), - 'servers': server_sql.select_servers(full=1), + 'group': roxywi_common.get_user_group(id=1), + 'sshs': sshs, + 'servers': servers, 'roles': sql.select_roles(), - 'timezones': pytz.all_timezones, - 'settings': sql.get_setting('', all=1), 'ldap_enable': sql.get_setting('ldap_enable'), 'services': service_sql.select_services(), - 'masters': server_sql.select_servers(get_master_servers=1), + 'masters': masters, 'guide_me': 1, - 'user_subscription': roxywi_common.return_user_subscription() + 'user_subscription': roxywi_common.return_user_subscription(), + 'user_roles': user_sql.select_user_roles_by_group(user_group), } return render_template('admin.html', **kwargs) @@ -203,6 +216,16 @@ def action_openvpn(action, openvpn): return f'error: Cannot {action} OpenVPN: {e}' +@bp.get('/settings') +@get_user_params() +def get_settings(): + kwargs = { + 'settings': sql.get_setting('', all=1), + 'timezones': pytz.all_timezones, + } + return render_template('include/admin_settings.html', **kwargs) + + @bp.post('/setting/') def update_settings(param): roxywi_auth.page_for_admin(level=2) diff --git a/app/routes/main/routes.py b/app/routes/main/routes.py index e01c3564..f66133a8 100644 --- a/app/routes/main/routes.py +++ b/app/routes/main/routes.py @@ -1,6 +1,5 @@ import os import sys -import pytz from flask import render_template, request, session, g, abort, jsonify from flask_login import login_required @@ -9,10 +8,7 @@ sys.path.append(os.path.join(sys.path[0], '/var/www/haproxy-wi/app')) from app import app, cache from app.routes.main import bp -import app.modules.db.sql as sql -import app.modules.db.cred as cred_sql import app.modules.db.user as user_sql -import app.modules.db.group as group_sql import app.modules.db.server as server_sql import app.modules.db.service as service_sql import app.modules.db.history as history_sql @@ -174,37 +170,6 @@ def service_history(service, server_ip): return render_template('history.html', **kwargs) - -@bp.route('/servers') -@login_required -@get_user_params() -def servers(): - roxywi_auth.page_for_admin(level=2) - - user_group = roxywi_common.get_user_group(id=1) - kwargs = { - 'h2': 1, - 'users': user_sql.select_users(group=user_group), - 'groups': group_sql.select_groups(), - 'servers': roxywi_common.get_dick_permit(virt=1, disable=0, only_group=1), - 'roles': sql.select_roles(), - 'sshs': cred_sql.select_ssh(group=user_group), - 'masters': server_sql.select_servers(get_master_servers=1, uuid=g.user_params['user_uuid']), - 'group': roxywi_common.get_user_group(id=1), - 'services': service_sql.select_services(), - 'timezones': pytz.all_timezones, - 'guide_me': 1, - 'settings': sql.get_setting('', all=1), - 'page': 'servers.py', - 'ldap_enable': sql.get_setting('ldap_enable'), - 'user_roles': user_sql.select_user_roles_by_group(user_group), - 'user_subscription': roxywi_common.return_user_subscription(), - 'lang': g.user_params['lang'] - } - - return render_template('servers.html', **kwargs) - - @bp.route('/internal/show_version') @cache.cached() def show_roxywi_version(): diff --git a/app/routes/metric/routes.py b/app/routes/metric/routes.py index 351ebb0f..f690585b 100644 --- a/app/routes/metric/routes.py +++ b/app/routes/metric/routes.py @@ -1,5 +1,8 @@ +import json +import time + import distro -from flask import render_template, request, jsonify, g +from flask import render_template, request, jsonify, g, Response, stream_with_context from flask_login import login_required from app.routes.metric import bp @@ -120,3 +123,38 @@ def show_http_metric(service, server_ip): return jsonify(metric.haproxy_http_metrics(server_ip, hostname, time_range)) return 'error: Wrong service' + + +@bp.route('////chart-stream') +@check_services +def chart_data(service, server_ip, is_http): + def get_chart_data(): + while True: + json_metric = {} + if service in ('nginx', 'apache', 'waf'): + chart_metrics = metric_sql.select_metrics(server_ip, service, time_range=1) + for i in chart_metrics: + json_metric['time'] = common.get_time_zoned_date(i[2], '%H:%M:%S') + json_metric['value'] = str(i[1]) + elif service == 'haproxy' and not is_http: + chart_metrics = metric_sql.select_metrics(server_ip, 'haproxy', time_range=1) + for i in chart_metrics: + json_metric['time'] = common.get_time_zoned_date(i[5], '%H:%M:%S') + json_metric['value'] = str(i[1]) + json_metric['value1'] = str(i[2]) + json_metric['value2'] = str(i[3]) + else: + chart_metrics = metric_sql.select_metrics(server_ip, 'http_metrics', time_range=1) + for i in chart_metrics: + json_metric['time'] = common.get_time_zoned_date(i[5], '%H:%M:%S') + json_metric['value'] = str(i[1]) + json_metric['value1'] = str(i[2]) + json_metric['value2'] = str(i[3]) + json_metric['value3'] = str(i[4]) + yield f"data:{json.dumps(json_metric)}\n\n" + time.sleep(60) + + response = Response(stream_with_context(get_chart_data()), mimetype="text/event-stream") + response.headers["Cache-Control"] = "no-cache" + response.headers["X-Accel-Buffering"] = "no" + return response diff --git a/app/static/css/style-6.3.9.css b/app/static/css/style-6.3.9.css index ce01cd31..8250b45b 100644 --- a/app/static/css/style-6.3.9.css +++ b/app/static/css/style-6.3.9.css @@ -1373,12 +1373,12 @@ label { margin-top: var(--indent); margin-left: var(--indent); } -.metrics-refresh { - text-align: right; - margin-top: -25px; - margin-right: 10px; - margin-bottom: 20px; -} +/*.metrics-refresh {*/ +/* text-align: right;*/ +/* margin-top: -25px;*/ +/* margin-right: 10px;*/ +/* margin-bottom: 20px;*/ +/*}*/ .ajaxwafstatus { padding-top: 2px; padding-left: 10px; diff --git a/app/static/js/metrics.js b/app/static/js/metrics.js index 81fc7ab4..35970fe7 100644 --- a/app/static/js/metrics.js +++ b/app/static/js/metrics.js @@ -1,10 +1,97 @@ +function return_service_chart_config() { + var config = { + type: 'line', + data: { + labels: [], + datasets: [ + { + normalized: true, + label: 'Connections', + data: [], + borderColor: 'rgba(75, 192, 192, 1)', + backgroundColor: 'rgba(75, 192, 192, 0.2)', + fill: true + } + ] + }, + options: { + animation: false, + maintainAspectRatio: false, + plugins: { + title: { + display: true, + text: '', + font: { + size: 20 + }, + legend: { + position: 'bottom' + } + }, + legend: { + display: true, + labels: { + color: 'rgb(255, 99, 132)', + font: { + size: '10', + family: 'BlinkMacSystemFont' + } + }, + position: 'bottom' + } + }, + scales: { + y: { + beginAtZero: true + }, + x: { + ticks: { + major: { + enabled: true, + fontStyle: 'bold' + }, + source: 'data', + autoSkip: true, + autoSkipPadding: 45, + maxRotation: 0 + } + } + } + } + } + return config; +} +function stream_chart(chart_id, service, service_ip, is_http=0) { + let random_sleep = getRandomArbitrary(500, 2000); + console.log(`stream_chart ${random_sleep}`); + sleep(random_sleep); + console.log('stop sleep') + const source = new EventSource(`/app/metrics/${service}/${service_ip}/${is_http}/chart-stream`); + source.onmessage = function (event) { + const data = JSON.parse(event.data); + if (chart_id.data.labels.length >= 30) { + chart_id.data.labels.shift(); + chart_id.data.datasets[0].data.shift(); + } + chart_id.data.labels.push(data.time); + chart_id.data.datasets[0].data.push(data.value); + if (service == 'haproxy') { + chart_id.data.datasets[1].data.push(data.value1); + chart_id.data.datasets[2].data.push(data.value2); + } + if (is_http) { + chart_id.data.datasets[3].data.push(data.value3); + } + chart_id.update(); + } +} function getHttpChartData(server) { var hide_http_metrics = localStorage.getItem('hide_http_metrics'); if (hide_http_metrics == 'disabled') { return false; } $.ajax({ - url: "/app/metrics/haproxy/" + server + "/http", + url: `/app/metrics/haproxy/${server}/http`, data: { time_range: $( "#time-range option:selected" ).val(), }, @@ -122,7 +209,7 @@ function renderHttpChart(data, labels, server) { }, }, }); - charts.push(myChart); + stream_chart(myChart, 'haproxy', server, 1); } function getChartData(server) { $.ajax({ @@ -226,7 +313,7 @@ function renderChart(data, labels, server) { } } }); - charts.push(myChart); + stream_chart(myChart, 'haproxy', server); } function getWafChartData(server) { $.ajax({ @@ -249,72 +336,24 @@ function renderServiceChart(data, labels, server, service) { // Удаление последнего пустого элемента в каждом массиве dataArray.pop(); + dataArray.pop(); + var label = labels.split(','); + label.pop(); + label.pop(); var ctx = document.getElementById(service + '_' + server).getContext('2d'); var additional_title = ''; if (service === 'waf') { additional_title = 'WAF '; } - var myChart = new Chart(ctx, { - type: 'line', - data: { - labels: labels.split(','), - datasets: [ - { - normalized: true, - label: 'Connections', - data: dataArray, - borderColor: 'rgba(75, 192, 192, 1)', - backgroundColor: 'rgba(75, 192, 192, 0.2)', - fill: true - } - ] - }, - options: { - animation: false, - maintainAspectRatio: false, - plugins: { - title: { - display: true, - text: additional_title + data[1], - font: { - size: 20 - }, - legend: { - position: 'bottom' - } - }, - legend: { - display: true, - labels: { - color: 'rgb(255, 99, 132)', - font: { - size: '10', - family: 'BlinkMacSystemFont' - } - }, - position: 'bottom' - } - }, - scales: { - y: { - beginAtZero: true - }, - x: { - ticks: { - major: { - enabled: true, - fontStyle: 'bold' - }, - source: 'data', - autoSkip: true, - autoSkipPadding: 45, - maxRotation: 0 - } - } - } - } - }); - charts.push(myChart); + config = return_service_chart_config(); + for (var i=0; i
{% include 'include/admin_users.html' %}
- + {% if g.user_params['role'] == 1 %}
@@ -68,6 +70,7 @@ {{lang.words.and}} {{lang.words.this2}} {{lang.words.article}} + {% endif %}
{% include 'include/admin_servers.html' %}
@@ -75,13 +78,11 @@
{% include 'include/admin_ssh.html' %}
+ +
-
- -
- {% include 'include/admin_settings.html' %} -
- +
+ {% if g.user_params['role'] == 1 %}
@@ -132,7 +133,8 @@
-
+
+ {% endif %} {% include 'include/admins_dialogs.html' %} -{% include 'include/intro/admin.html' %} +{% if g.user_params['role'] == 1 %} + {% include 'include/intro/admin.html' %} +{% else %} + {% include 'include/intro/servers.html' %} +{% endif %} {% include 'include/intro/js_script.html' %} {% endblock %} diff --git a/app/templates/include/admin_servers.html b/app/templates/include/admin_servers.html index 587f0929..a5e8d181 100644 --- a/app/templates/include/admin_servers.html +++ b/app/templates/include/admin_servers.html @@ -6,7 +6,7 @@ - {% if page != "servers.py" %} + {% if g.user_params['role'] == 1 %} {% endif %} @@ -33,7 +33,7 @@ {% endif %} - {% for server in servers %} + {% for server in servers %} - {% if page != "servers.py" %} + {% if g.user_params['role'] == 1 %} - {% endfor %} + {% endfor %} {% if not adding %}
{{lang.words.name|title()}} IP {{lang.words.port|title()}}{{lang.words.group|title()}}{{lang.words.enabled|title()}}
@@ -48,7 +48,7 @@ {% set id = 'port-' + server.0|string() %} {{ input(id, value=server.10, type='number', style='width: 50px;') }}
diff --git a/app/templates/include/admin_settings.html b/app/templates/include/admin_settings.html index 2cf76424..affc831b 100644 --- a/app/templates/include/admin_settings.html +++ b/app/templates/include/admin_settings.html @@ -1,3 +1,5 @@ +{% from 'include/input_macros.html' import input, checkbox %} +{% import 'languages/'+lang|default('en')+'.html' as lang %} {% set sections_name = { 'rabbitmq': 'RabbitMQ', 'nginx': 'NGINX', @@ -17,7 +19,7 @@ {% set section = namespace(section='') %} {% for set in settings %} - {% if page == "servers.py" and (set.section in ('monitoring', 'rabbitmq', 'mail', 'smon')) %} + {% if g.user_params['role'] == 2 and (set.section in ('monitoring', 'rabbitmq', 'mail', 'smon')) %} {% else %} {% if section.section|string() != set.section|string() %} @@ -33,7 +35,7 @@ {% endif %} {% set section.section = set.section %} - {% if page == "servers.py" and (set.param == 'proxy') %} + {% if g.user_params['role'] == 2 and (set.param == 'proxy') %} {% else %} diff --git a/app/templates/include/admin_ssh.html b/app/templates/include/admin_ssh.html index c0979221..74587ab4 100644 --- a/app/templates/include/admin_ssh.html +++ b/app/templates/include/admin_ssh.html @@ -6,7 +6,7 @@ SSH {{lang.words.key}} - {% if page != "servers.py" %} + {% if g.user_params['role'] == 1 %} {{lang.words.group|title()}} {% endif %} @@ -27,7 +27,7 @@ {% endif %} - {% if page != "servers.py" %} + {% if g.user_params['role'] == 1 %} {% for r in roles %} {% for user_role in user_roles %} - {% if r.role_id|int() != 1 and user.user_id == user_role.user_id %} - {% if user_role.user_role_id == r.role_id %} + {% if r.role_id|int() != 1 and user.user_id|string() == user_role.user_id|string() %} + {% if user_role.user_role_id|string() == r.role_id|string() %} {% else %} diff --git a/app/templates/include/admins_dialogs.html b/app/templates/include/admins_dialogs.html index dcf37005..ed4f8f25 100644 --- a/app/templates/include/admins_dialogs.html +++ b/app/templates/include/admins_dialogs.html @@ -1,4 +1,4 @@ -{% if page != "servers.py" %} +{% if g.user_params['role'] == 1 %}