From e8cb5366046a2267ca828c45d98da05079e3b00b Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Fri, 7 Oct 2022 17:40:45 +0300 Subject: [PATCH] v6.2.1.0 Changelog: https://roxy-wi.org/changelog#6_2_1 --- app/create_db.py | 12 ++++- app/options.py | 51 ++++++++++++------- app/scripts/ansible/roles/nginx_geoip.yml | 11 ++++ .../ansible/roles/nginx_geoip/tasks/main.yml | 43 ++++++++++++++++ .../roles/nginx_geoip/templates/geoip.sh.j2 | 9 ++++ ...tall_geoip.sh => install_haproxy_geoip.sh} | 6 +-- app/scripts/install_nginx_geoip.sh | 46 +++++++++++++++++ app/sql.py | 2 +- app/templates/servers.html | 8 +++ inc/users.js | 49 +++++++++++------- 10 files changed, 195 insertions(+), 42 deletions(-) create mode 100644 app/scripts/ansible/roles/nginx_geoip.yml create mode 100644 app/scripts/ansible/roles/nginx_geoip/tasks/main.yml create mode 100644 app/scripts/ansible/roles/nginx_geoip/templates/geoip.sh.j2 rename app/scripts/{install_geoip.sh => install_haproxy_geoip.sh} (88%) create mode 100644 app/scripts/install_nginx_geoip.sh diff --git a/app/create_db.py b/app/create_db.py index cc9bd1cc..28f05f13 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -955,9 +955,18 @@ def update_db_v_6_1_4(): pass +def update_db_v_6_2_1(): + try: + Setting.update(section='main').where(Setting.param == 'maxmind_key').execute() + except Exception as e: + print("An error occurred:", e) + else: + print("Updating... DB has been updated to version 6.2.1.0") + + def update_ver(): try: - Version.update(version='6.2.0.0').execute() + Version.update(version='6.2.1.0').execute() except Exception: print('Cannot update version') @@ -992,6 +1001,7 @@ def update_all(): update_db_v_6_1_0() update_db_v_6_1_3() update_db_v_6_1_4() + update_db_v_6_2_1() update_ver() diff --git a/app/options.py b/app/options.py index a7ed5cb0..e17bff51 100644 --- a/app/options.py +++ b/app/options.py @@ -677,7 +677,7 @@ if act == "overviewHapservers": try: print(funct.ssh_command(serv, commands)) except Exception as e: - print('error: Cannot get last date ' + str(e)) + print(f'error: Cannot get last date {e} for server {serv}') if act == "overview": import asyncio @@ -719,11 +719,19 @@ if act == "overview": if keepalived == 1: command = ["ps ax |grep keepalived|grep -v grep|wc -l|tr -d '\n'"] - keepalived_process = funct.ssh_command(serv2, command) + try: + keepalived_process = funct.ssh_command(serv2, command) + except Exception as e: + print(f'{e} for server {serv2}') + sys.exit() if waf_len >= 1: command = ["ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l"] - waf_process = funct.ssh_command(serv2, command) + try: + waf_process = funct.ssh_command(serv2, command) + except Exception as e: + print(f'{e} for server {serv2}') + sys.exit() server_status = (serv1, serv2, @@ -3171,40 +3179,45 @@ if form.getvalue('viewFirewallRules') is not None: if form.getvalue('geoipserv') is not None: serv = form.getvalue('geoipserv') - haproxy_dir = sql.get_setting('haproxy_dir') + service = form.getvalue('geoip_service') + if service in ('haproxy', 'nginx'): + service_dir = funct.return_nice_path(sql.get_setting(f'{service}_dir')) - cmd = ["ls " + haproxy_dir + "/geoip/"] - print(funct.ssh_command(serv, cmd)) + cmd = ["ls " + service_dir + "geoip/"] + print(funct.ssh_command(serv, cmd)) + else: + print('warning: select a server and service first') if form.getvalue('geoip_install'): - serv = form.getvalue('geoip_install') - geoip_update = form.getvalue('geoip_update') + serv = funct.is_ip_or_dns(form.getvalue('geoip_install')) + geoip_update = funct.checkAjaxInput(form.getvalue('geoip_update')) + service = form.getvalue('geoip_service') proxy = sql.get_setting('proxy') maxmind_key = sql.get_setting('maxmind_key') - haproxy_dir = sql.get_setting('haproxy_dir') - script = 'install_geoip.sh' - ssh_port = '22' ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = funct.return_ssh_keys_path(serv) + if service in ('haproxy', 'nginx'): + service_dir = funct.return_nice_path(sql.get_setting(f'{service}_dir')) + script = f'install_{service}_geoip.sh' + else: + print('warning: select a server and service first') + sys.exit() + if ssh_enable == 0: ssh_key_name = '' - servers = sql.select_servers(server=serv) - for server in servers: - ssh_port = str(server[10]) + ssh_port = [ str(server[10]) for server in sql.select_servers(server=serv) ] if proxy is not None and proxy != '' and proxy != 'None': proxy_serv = proxy else: proxy_serv = '' - os.system("cp scripts/%s ." % script) + os.system(f"cp scripts/{script} .") commands = [ - "chmod +x " + script + " && ./" + script + " PROXY=" + proxy_serv + " SSH_PORT=" + ssh_port - + " UPDATE=" + str(geoip_update) + " maxmind_key=" + maxmind_key + " haproxy_dir=" + haproxy_dir - + " HOST=" + str(serv) + " USER=" + str(ssh_user_name) + " PASS=" + str(ssh_user_password) - + " KEY=" + str(ssh_key_name) + f"chmod +x {script} && ./{script} PROXY={proxy_serv} SSH_PORT={ssh_port[0]} UPDATE={geoip_update} maxmind_key={maxmind_key} " + f"service_dir={service_dir} HOST={serv} USER={ssh_user_name} PASS={ssh_user_password} KEY={ssh_key_name}" ] output, error = funct.subprocess_execute(commands[0]) diff --git a/app/scripts/ansible/roles/nginx_geoip.yml b/app/scripts/ansible/roles/nginx_geoip.yml new file mode 100644 index 00000000..3bd57106 --- /dev/null +++ b/app/scripts/ansible/roles/nginx_geoip.yml @@ -0,0 +1,11 @@ +--- +- name: Install NGINX GeoIP + hosts: "{{ variable_host }}" + become: yes + become_method: sudo + gather_facts: yes + roles: + - role: nginx_geoip + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" diff --git a/app/scripts/ansible/roles/nginx_geoip/tasks/main.yml b/app/scripts/ansible/roles/nginx_geoip/tasks/main.yml new file mode 100644 index 00000000..57b2fe8d --- /dev/null +++ b/app/scripts/ansible/roles/nginx_geoip/tasks/main.yml @@ -0,0 +1,43 @@ +--- +- name: Set SSH port + set_fact: + ansible_port: "{{SSH_PORT}}" + +- name: Creates directory + file: + path: "{{nginx_dir}}/geoip" + state: directory + +- name: Creates directory + file: + path: "{{nginx_dir}}/scripts" + state: directory + +- name: Install wget + package: + name: "{{ item }}" + state: present + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" + with_items: + - git + - python3 + +- name: Copy GeoIP script in place. + template: + src: geoip.sh.j2 + dest: "{{nginx_dir}}/scripts/geoip.sh" + mode: 0777 + +- name: Execute the script + command: "{{nginx_dir}}/scripts/geoip.sh" + +- name: Update geoip every Wednesday + cron: + name: "Update geoip" + minute: "0" + hour: "01" + weekday: "3" + job: "{{nginx_dir}}/scripts/geoip.sh" + when: UPDATE == "1" diff --git a/app/scripts/ansible/roles/nginx_geoip/templates/geoip.sh.j2 b/app/scripts/ansible/roles/nginx_geoip/templates/geoip.sh.j2 new file mode 100644 index 00000000..1061c64e --- /dev/null +++ b/app/scripts/ansible/roles/nginx_geoip/templates/geoip.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/bash + +cd {{nginx_dir}}/scripts +git clone https://github.com/sherpya/geolite2legacy.git || true +cd geolite2legacy +wget "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key={{maxmind_key}}&suffix=zip" -qO geoip2Country.zip + +python3 geolite2legacy.py -i geoip2Country.zip -o GeoIP.dat +mv GeoIP.dat {{nginx_dir}}/geoip/ diff --git a/app/scripts/install_geoip.sh b/app/scripts/install_haproxy_geoip.sh similarity index 88% rename from app/scripts/install_geoip.sh rename to app/scripts/install_haproxy_geoip.sh index be29174b..438b41e6 100644 --- a/app/scripts/install_geoip.sh +++ b/app/scripts/install_haproxy_geoip.sh @@ -8,7 +8,7 @@ do PROXY) PROXY=${VALUE} ;; UPDATE) UPDATE=${VALUE} ;; maxmind_key) maxmind_key=${VALUE} ;; - haproxy_dir) haproxy_dir=${VALUE} ;; + service_dir) service_dir=${VALUE} ;; HOST) HOST=${VALUE} ;; USER) USER=${VALUE} ;; PASS) PASS=${VALUE} ;; @@ -34,9 +34,9 @@ if [[ $maxmind_key == "" ]]; then fi if [[ $KEY == "" ]]; then - ansible-playbook $PWD/roles/geoip.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE haproxy_dir=$haproxy_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST + ansible-playbook $PWD/roles/geoip.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE haproxy_dir=$service_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST else - ansible-playbook $PWD/roles/geoip.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE haproxy_dir=$haproxy_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST + ansible-playbook $PWD/roles/geoip.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE haproxy_dir=$service_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST fi if [ $? -gt 0 ] diff --git a/app/scripts/install_nginx_geoip.sh b/app/scripts/install_nginx_geoip.sh new file mode 100644 index 00000000..3cf3859c --- /dev/null +++ b/app/scripts/install_nginx_geoip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +for ARGUMENT in "$@" +do + KEY=$(echo "$ARGUMENT" | cut -f1 -d=) + VALUE=$(echo "$ARGUMENT" | cut -f2 -d=) + + case "$KEY" in + PROXY) PROXY=${VALUE} ;; + UPDATE) UPDATE=${VALUE} ;; + maxmind_key) maxmind_key=${VALUE} ;; + service_dir) service_dir=${VALUE} ;; + HOST) HOST=${VALUE} ;; + USER) USER=${VALUE} ;; + PASS) PASS=${VALUE} ;; + KEY) KEY=${VALUE} ;; + SSH_PORT) SSH_PORT=${VALUE} ;; + *) + esac +done + +export ANSIBLE_HOST_KEY_CHECKING=False +export ANSIBLE_DISPLAY_SKIPPED_HOSTS=False +export ACTION_WARNINGS=False +export LOCALHOST_WARNING=False +export COMMAND_WARNINGS=False + +PWD=$(pwd) +PWD=$PWD/scripts/ansible/ +echo "$HOST ansible_port=$SSH_PORT" > $PWD/$HOST + +if [[ $maxmind_key == "" ]]; then + echo "error: the Maxmind key cannot be empty" + exit 1 +fi + +if [[ $KEY == "" ]]; then + ansible-playbook $PWD/roles/nginx_geoip.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE nginx_dir=$service_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST +else + ansible-playbook $PWD/roles/nginx_geoip.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY UPDATE=$UPDATE nginx_dir=$service_dir maxmind_key=$maxmind_key SSH_PORT=$SSH_PORT" -i $PWD/$HOST +fi + +if [ $? -gt 0 ] +then + echo "error: Cannot download GeoLite2 database" +fi +rm -f $PWD/$HOST diff --git a/app/sql.py b/app/sql.py index 4ee25a53..2a92a644 100755 --- a/app/sql.py +++ b/app/sql.py @@ -908,7 +908,7 @@ def select_ssh(**kwargs): elif kwargs.get("serv") is not None: query = Cred.select().join(Server, on=(Cred.id == Server.cred)).where(Server.ip == kwargs.get('serv')) elif kwargs.get("group") is not None: - query = Cred.select() + query = Cred.select().where(Cred.groups == kwargs.get("group")) else: query = Cred.select() try: diff --git a/app/templates/servers.html b/app/templates/servers.html index a719b4ab..6121d6e7 100644 --- a/app/templates/servers.html +++ b/app/templates/servers.html @@ -128,6 +128,7 @@

Install GeoLite2

Server + Service Current installation Updating @@ -142,6 +143,13 @@ {% endfor %} + + + {{ checkbox('updating_geoip', title="Update the database?", checked='checked') }} diff --git a/inc/users.js b/inc/users.js index 3986656b..8926cedf 100644 --- a/inc/users.js +++ b/inc/users.js @@ -746,24 +746,15 @@ $( function() { $('#hide_country_codes').hide(); }); $( "#geoipserv" ).on('selectmenuchange',function() { - $.ajax( { - url: "options.py", - data: { - geoipserv: $('#geoipserv option:selected').val(), - token: $('#token').val() - }, - type: "POST", - success: function( data ) { - data = data.replace(/^\s+|\s+$/g,''); - if(data.indexOf('No such file or directory') != '-1') { - $('#cur_geoip').text('GeoLite2 has not installed'); - $('#geoip_install').show(); - } else { - $('#cur_geoip').text('GeoLite2 has already installed'); - $('#geoip_install').hide(); - } - } - } ); + if($('#geoip_service option:selected').val() != '------') { + checkGeoipInstallation(); + } + + }); + $( "#geoip_service" ).on('selectmenuchange',function() { + if($('#geoipserv option:selected').val() != '------') { + checkGeoipInstallation(); + } }); $( "#geoip_install" ).click(function() { var updating_geoip = 0; @@ -775,6 +766,7 @@ $( function() { url: "options.py", data: { geoip_install: $('#geoipserv option:selected').val(), + geoip_service: $('#geoip_service option:selected').val(), geoip_update: updating_geoip, token: $('#token').val() }, @@ -2845,3 +2837,24 @@ function checkEmail() { } }); } +function checkGeoipInstallation() { + $.ajax( { + url: "options.py", + data: { + geoipserv: $('#geoipserv option:selected').val(), + geoip_service: $('#geoip_service option:selected').val(), + token: $('#token').val() + }, + type: "POST", + success: function( data ) { + data = data.replace(/^\s+|\s+$/g,''); + if(data.indexOf('No such file or directory') != '-1') { + $('#cur_geoip').text('GeoLite2 has not installed'); + $('#geoip_install').show(); + } else { + $('#cur_geoip').text('GeoLite2 has already installed'); + $('#geoip_install').hide(); + } + } + } ); +}