From 1ae61a35bf1f1b3fc3c464779d5e5afcfdaf3518 Mon Sep 17 00:00:00 2001 From: Pavel Loginov Date: Tue, 2 Aug 2022 11:53:32 +0300 Subject: [PATCH] v6.1.3.0 Change log: https://roxy-wi.org/changelog.py#6_1_3 --- README.md | 109 ++++----- app/create_db.py | 20 +- app/db_model.py | 15 +- app/funct.py | 62 +++-- app/options.py | 178 ++++++++++---- app/scripts/ansible/roles/waf/tasks/main.yml | 159 ++++--------- app/scripts/ansible/roles/waf_nginx.yml | 11 + .../ansible/roles/waf_nginx/defaults/main.yml | 1 + .../ansible/roles/waf_nginx/handlers/main.yml | 3 + .../ansible/roles/waf_nginx/tasks/main.yml | 224 ++++++++++++++++++ .../roles/waf_nginx/templates/waf.conf.j2 | 31 +++ app/scripts/waf_nginx.sh | 43 ++++ app/sql.py | 136 ++++++++++- app/templates/ajax/overivewWaf.html | 24 +- app/templates/base.html | 3 +- app/templates/waf.html | 50 +++- app/waf.py | 54 ++++- inc/overview.js | 28 ++- inc/script.js | 15 +- inc/users.js | 2 +- inc/waf.js | 81 ++++++- 21 files changed, 994 insertions(+), 255 deletions(-) create mode 100644 app/scripts/ansible/roles/waf_nginx.yml create mode 100644 app/scripts/ansible/roles/waf_nginx/defaults/main.yml create mode 100644 app/scripts/ansible/roles/waf_nginx/handlers/main.yml create mode 100644 app/scripts/ansible/roles/waf_nginx/tasks/main.yml create mode 100644 app/scripts/ansible/roles/waf_nginx/templates/waf.conf.j2 create mode 100644 app/scripts/waf_nginx.sh diff --git a/README.md b/README.md index 6c5fd255..e64b485c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ![alt text](https://roxy-wi.org/inc/images/logo_menu.png "Logo") +# ![alt text](https://roxy-wi.org/static/images/logo_menu.png "Logo") Web interface(user-friendly web GUI, alerting, monitoring and secure) for managing HAProxy, Nginx and Keepalived servers. Leave your [feedback](https://github.com/hap-wi/roxy-wi/issues) # Get involved @@ -9,75 +9,76 @@ Web interface(user-friendly web GUI, alerting, monitoring and secure) for managi # Demo site [Demo site](https://demo.roxy-wi.org) Login/password: admin/admin. Server resets every hour. -![alt text](https://roxy-wi.org/inc/images/viewstat.png "HAProxy state page") +![alt text](https://roxy-wi.org/static/images/viewstat.png "HAProxy state page") # Features: 1. Installing and updating HAProxy, Nginx and Keepalived with Roxy-WI as a system service -1. Installing and updating HAProxy and Nginx with Roxy-WI as a Docker service -2. Installing and updating Grafana, Prometheus servers with Roxy-WI -3. Installing and updating HAProxy and Nginx exporters with Roxy-WI -4. Server provisioning on AWS, DigitalOcean and G-Core Labs -5. Downloading, updating and formatting GeoIP to the acceptable format for HAProxy with Roxy-WI -6. Dynamic change of Maxconn, Black/white lists and backend's IP address and port with saving changes to the config file -7. Configuring HAProxy, Nginx, Apache and Keepalived in a jiffy with Roxy-WI -8. Viewing and analysing the status of all Frontend/backend servers via Roxy-WI from a single control panel -9. Enabling/disabling servers through stats page without rebooting HAProxy -1. Viewing/Analysing HAProxy, Nginx and Apache logs right from the Roxy-WI web interface -1. Creating and visualizing the HAProxy workflow from Web Ui -1. Pushing Your changes to your HAProxy, Nginx, Apache and Keepalived servers with a single click via the web interface -1. Getting info on past changes, evaluating your config files and restoring the previous stable config at any time with a single click right from Web interface -1. Adding/Editing Frontend or backend servers via the web interface with a click -1. Editing the config of HAProxy, Nginx, Apache and Keepalived and push ingchanges to All Master/Slave servers by a single click -1. Adding Multiple server to ensure the Config Sync between servers -1. Managing the ports assigned to Frontend automatically -1. Evaluating the changes of recent configs pushed to HAProxy, Nginx and Keepalived instances right from the Web UI -1. Multiple User Roles support for privileged based Viewing and editing of Config -1. Creating Groups and adding/removing servers to ensure the proper identification for your HAProxy and Nginx Clusters -1. Sending notifications from Roxy-WI via Telegram, Slack and via the web interface -1. Supporting high Availability to ensure uptime to all Master slave servers configured -1. Support of SSL (including Let's Encrypt) -1. Support of SSH Key for managing multiple HAProxy, Nginx, Apache and Keepalived Servers straight from Roxy-WI -1. SYN flood protect -1. Alerting about changes of the state of HAProxy backends -1. Alerting about the state of HAProxy, Nginx, Apache and Keepalived service -1. Gathering metrics for incoming connections -1. Web acceleration settings -1. Firewall for web application -1. LDAP support -1. Keep active HAProxy, Nginx and Keepalived services -1. Possibility to hide parts of the config with tags for users with "guest" role: "HideBlockStart" and "HideBlockEnd" -1. Mobile-ready design -1. Simple port monitoring (SMON) -1. Backup HAProxy, Nginx and Keepalived config files through Roxy-WI -1. Managing OpenVPN3 as a client via Roxy-WI +2. Installing and updating HAProxy and Nginx with Roxy-WI as a Docker service +3. Installing and updating Grafana, Prometheus servers with Roxy-WI +4. Installing and updating HAProxy and Nginx exporters with Roxy-WI +5. Server provisioning on AWS, DigitalOcean and G-Core Labs +6. Downloading, updating and formatting GeoIP to the acceptable format for HAProxy with Roxy-WI +7. Dynamic change of Maxconn, Black/white lists and backend's IP address and port with saving changes to the config file +8. Configuring HAProxy, Nginx, Apache and Keepalived in a jiffy with Roxy-WI +9. Viewing and analysing the status of all Frontend/backend servers via Roxy-WI from a single control panel +10. Enabling/disabling servers through stats page without rebooting HAProxy +11. Viewing/Analysing HAProxy, Nginx and Apache logs right from the Roxy-WI web interface +12. Creating and visualizing the HAProxy workflow from Web Ui +13. Pushing Your changes to your HAProxy, Nginx, Apache and Keepalived servers with a single click via the web interface +14. Getting info on past changes, evaluating your config files and restoring the previous stable config at any time with a single click right from Web interface +15. Adding/Editing Frontend or backend servers via the web interface with a click +16. Editing the config of HAProxy, Nginx, Apache and Keepalived and push ingchanges to All Master/Slave servers by a single click +17. Adding Multiple server to ensure the Config Sync between servers +18. Managing the ports assigned to Frontend automatically +19. Evaluating the changes of recent configs pushed to HAProxy, Nginx, Apache and Keepalived instances right from the Web UI +20. Multiple User Roles support for privileged based Viewing and editing of Config +21. Creating Groups and adding/removing servers to ensure the proper identification for your HAProxy and Nginx Clusters +22. Sending notifications from Roxy-WI via Telegram, Slack, Email and via the web interface +23. Supporting high Availability to ensure uptime to all Master slave servers configured +24. Support of SSL (including Let's Encrypt) +25. Support of SSH Key for managing multiple HAProxy, Nginx, Apache and Keepalived Servers straight from Roxy-WI +26. SYN flood protect +27. Alerting about changes of the state of HAProxy backends +28. Alerting about the state of HAProxy, Nginx, Apache and Keepalived service +29. Gathering metrics for incoming connections +30. Web acceleration settings +31. Firewall for web application (WAF) +32. LDAP support +33. Keep active HAProxy, Nginx, Apache and Keepalived services +34. Possibility to hide parts of the config with tags for users with "guest" role: "HideBlockStart" and "HideBlockEnd" +35. Mobile-ready design +36. Simple port monitoring (SMON) +37. Backup HAProxy, Nginx, Apache and Keepalived config files through Roxy-WI +38. Managing OpenVPN3 as a client via Roxy-WI -![alt text](https://Roxy-WI.org/inc/images/roxy-wi-metrics.png "Merics") +![alt text](https://Roxy-WI.org/static/images/roxy-wi-metrics.png "Merics") # Install ## RPM -### Read instruction on the official [site](https://roxy-wi.org/installation.py#rpm) +### Read instruction on the official [site](https://roxy-wi.org/installation#rpm) ## DEB -### Read instruction on the official [site](https://roxy-wi.org/installation.py#deb) +### Read instruction on the official [site](https://roxy-wi.org/installation#deb) ## Manual install -### Read instruction on the official [site](https://roxy-wi.org/installation.py#manual) +### Read instruction on the official [site](https://roxy-wi.org/installation#manual) # OS support Roxy-WI supports the following OSes: 1. EL7(RPM installation and manual installation). It must be "Infrastructure Server" at least. x86_64 only 2. EL8(RPM installation and manual installation). It must be "Infrastructure Server" at least. x86_64 only -3. Amazon Linux 2(RPM installation and manual installation). x86_64 only -4. Ubuntu(DEB installation and manual installation). x86_64 only -5. Other Linux distributions (manual installation only). x86_64 only +3. EL9(RPM installation and manual installation). It must be "Infrastructure Server" at least. x86_64 only +4. Amazon Linux 2(RPM installation and manual installation). x86_64 only +5. Ubuntu(DEB installation and manual installation). x86_64 only +6. Other Linux distributions (manual installation only). x86_64 only -![alt text](https://roxy-wi.org/inc/images/smon_dashboard.png "SMON area") +![alt text](https://roxy-wi.org/static/images/smon_dashboard.png "SMON area") # Database support @@ -85,21 +86,21 @@ Default Roxy-WI use Sqlite, if you want use MySQL enable in config, and create d ### For MySQL support: -### Read instruction on the official [site](https://roxy-wi.org/installation.py#database) +### Read instruction on the official [site](https://roxy-wi.org/installation#database) -![alt text](https://roxy-wi.org/inc/images/roxy-wi-overview.webp "Overview page") +![alt text](https://roxy-wi.org/static/images/roxy-wi-overview.webp "Overview page") # Settings Login https://roxy-wi-server/users.py, and add: users, groups and servers. Default: admin/admin -### Read instruction on the official [site](https://roxy-wi.org/settings.py) +### Read instruction on the official [site](https://roxy-wi.org/settings) -![alt text](https://roxy-wi.org/inc/images/hapwi_overview.webp "HAProxy server overview page") +![alt text](https://roxy-wi.org/static/images/hapwi_overview.webp "HAProxy server overview page") -![alt text](https://roxy-wi.org/inc/images/add.png "Add proxy page") +![alt text](https://roxy-wi.org/static/images/add.png "Add proxy page") @@ -122,8 +123,8 @@ Do this: $ cd /var/www/haproxy-wi/app $ ./create_db.py ``` -and check executeble py files +and check executable py files If you see plain text, check section "Directory" in httpd conf -[Read more](https://roxy-wi.org/troubleshooting.py) +[Read more](https://roxy-wi.org/troubleshooting) diff --git a/app/create_db.py b/app/create_db.py index a1e63233..48db7af8 100644 --- a/app/create_db.py +++ b/app/create_db.py @@ -943,8 +943,24 @@ def update_db_v_6_1_0(**kwargs): print("An error occurred:", e) +def update_db_v_6_1_3(**kwargs): + cursor = conn.cursor() + sql = list() + sql.append("ALTER TABLE `waf_rules` ADD COLUMN service VARCHAR ( 64 ) DEFAULT 'haproxy'") + sql.append("ALTER TABLE `waf_rules` drop CONSTRAINT serv") + sql.append("ALTER TABLE `waf_rules` ADD CONSTRAINT UNIQUE (serv, rule_name, service)") + for i in sql: + try: + cursor.execute(i) + except Exception: + pass + else: + if kwargs.get('silent') != 1: + print('Updating... DB has been updated to version 6.1.3.0') + + def update_ver(): - query = Version.update(version='6.1.2.0') + query = Version.update(version='6.1.3.0') try: query.execute() except Exception: @@ -971,6 +987,7 @@ def update_all(): update_db_v_6_0() update_db_v_6_0_1() update_db_v_6_1_0() + update_db_v_6_1_3() update_ver() @@ -993,6 +1010,7 @@ def update_all_silent(): update_db_v_5_4_3_1(silent=1) update_db_v_6_0(silent=1) update_db_v_6_0_1(silent=1) + update_db_v_6_1_3(silent=1) update_ver() diff --git a/app/db_model.py b/app/db_model.py index ffc11c96..f48da0ef 100644 --- a/app/db_model.py +++ b/app/db_model.py @@ -280,10 +280,11 @@ class WafRules(BaseModel): rule_file = CharField() desc = TextField(null=True) en = IntegerField(constraints=[SQL('DEFAULT 1')]) + service = CharField(constraints=[SQL('haproxy')]) class Meta: table_name = 'waf_rules' - constraints = [SQL('UNIQUE (serv, rule_name)')] + constraints = [SQL('UNIQUE (serv, rule_name, service)')] class PortScannerSettings(BaseModel): @@ -550,10 +551,20 @@ class CheckerSetting(BaseModel): constraints = [SQL('UNIQUE (server_id, service_id)')] +class WafNginx(BaseModel): + id = AutoField() + server_id = ForeignKeyField(Server, on_delete='Cascade') + + class Meta: + table_name = 'waf_nginx' + constraints = [SQL('UNIQUE (server_id)')] + + def create_tables(): with conn: conn.create_tables([User, Server, Role, Telegram, Slack, UUID, Token, ApiToken, Groups, UserGroups, ConfigVersion, Setting, Cred, Backup, Metrics, WafMetrics, Version, Option, SavedServer, Waf, ActionHistory, PortScannerSettings, PortScannerPorts, PortScannerHistory, ProvidersCreds, ServiceSetting, ProvisionedServers, MetricsHttpStatus, SMON, WafRules, Alerts, GeoipCodes, NginxMetrics, - SystemInfo, Services, UserName, GitSetting, CheckerSetting, ApacheMetrics, ProvisionParam]) + SystemInfo, Services, UserName, GitSetting, CheckerSetting, ApacheMetrics, ProvisionParam, + WafNginx]) diff --git a/app/funct.py b/app/funct.py index 72578bdc..308d7c6e 100644 --- a/app/funct.py +++ b/app/funct.py @@ -448,7 +448,10 @@ def get_config(server_ip, cfg, **kwargs): ): config_path = kwargs.get('config_file_name') elif kwargs.get("waf") or kwargs.get("service") == 'waf': - config_path = sql.get_setting('haproxy_dir') + '/waf/rules/' + kwargs.get("waf_rule_file") + if kwargs.get("waf") == 'haproxy': + config_path = sql.get_setting('haproxy_dir') + '/waf/rules/' + kwargs.get("waf_rule_file") + elif kwargs.get("waf") == 'nginx': + config_path = sql.get_setting('nginx_dir') + '/waf/rules/' + kwargs.get("waf_rule_file") else: config_path = sql.get_setting('haproxy_config_path') @@ -783,14 +786,46 @@ def waf_install(server_ip): output, error = subprocess_execute(commands[0]) if show_installation_output(error, output, service): - ssh_command(server_ip, commands, print_out="1") - sql.insert_waf_metrics_enable(server_ip, "0") sql.insert_waf_rules(server_ip) os.system("rm -f %s" % script) +def waf_nginx_install(server_ip): + import sql + script = "waf_nginx.sh" + proxy = sql.get_setting('proxy') + nginx_dir = sql.get_setting('nginx_dir') + service = ' WAF' + ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = return_ssh_keys_path(server_ip) + ssh_port = '22' + + if ssh_enable == 0: + ssh_key_name = '' + + os.system("cp scripts/%s ." % script) + + if proxy is not None and proxy != '' and proxy != 'None': + proxy_serv = proxy + else: + proxy_serv = '' + + commands = [ + "chmod +x " + script + " && ./" + script + " PROXY=" + proxy_serv + " NGINX_PATH=" + nginx_dir + + " SSH_PORT=" + ssh_port + " HOST=" + server_ip + + " USER=" + ssh_user_name + " PASS='" + ssh_user_password + "' KEY=" + ssh_key_name + ] + + output, error = subprocess_execute(commands[0]) + + if show_installation_output(error, output, service): + sql.insert_nginx_waf_rules(server_ip) + sql.insert_waf_nginx_server(server_ip) + + os.system("rm -f %s" % script) + + def install_nginx(server_ip, **kwargs): import sql script = "install_nginx.sh" @@ -1541,30 +1576,25 @@ def check_ver(): return sql.get_ver() -def check_new_version(**kwargs): +def check_new_version(service): import requests import sql current_ver = check_ver() proxy = sql.get_setting('proxy') res = '' - if kwargs.get('service'): - last_ver = '_' + kwargs.get('service') - else: - last_ver = '' - user_name = sql.select_user_name() try: if proxy is not None and proxy != '' and proxy != 'None': proxy_dict = {"https": proxy, "http": proxy} - response = requests.get('https://roxy-wi.org/update.py?last_ver' + last_ver + '=1', timeout=1, proxies=proxy_dict) - requests.get('https://roxy-wi.org/update.py?ver_send=' + current_ver, timeout=1, proxies=proxy_dict) - response_status = requests.get('https://roxy-wi.org/update.py?user_name=' + user_name, timeout=1, proxies=proxy_dict) + response = requests.get(f'https://roxy-wi.org/version/get/{service}', timeout=1, proxies=proxy_dict) + requests.get(f'https://roxy-wi.org/version/send/{current_ver}', timeout=1, proxies=proxy_dict) + response_status = requests.get(f'https://roxy-wi.org/user-name/{user_name}', timeout=1, proxies=proxy_dict) else: - response = requests.get('https://roxy-wi.org/update.py?last_ver' + last_ver + '=1', timeout=1) - requests.get('https://roxy-wi.org/update.py?ver_send=' + current_ver, timeout=1) - response_status = requests.get('https://roxy-wi.org/update.py?user_name=' + user_name, timeout=1) + response = requests.get(f'https://roxy-wi.org/version/get/{service}', timeout=1) + requests.get(f'https://roxy-wi.org/version/send/{current_ver}', timeout=1) + response_status = requests.get(f'https://roxy-wi.org/user-name/{user_name}', timeout=1) res = response.content.decode(encoding='UTF-8') try: @@ -1595,7 +1625,7 @@ def versions(): current_ver_without_dots = 0 try: - new_ver = check_new_version() + new_ver = check_new_version('roxy-wi') new_ver_without_dots = new_ver.split('.') new_ver_without_dots = ''.join(new_ver_without_dots) new_ver_without_dots = new_ver_without_dots.replace('\n', '') diff --git a/app/options.py b/app/options.py index cbf24d12..55f88dca 100644 --- a/app/options.py +++ b/app/options.py @@ -41,6 +41,9 @@ try: except ValueError: print('error: Your token is not valid') sys.exit() +except Exception: + print('error: There is no token') + sys.exit() if not sql.check_token_exists(token): print('error: Your token has been expired') @@ -504,10 +507,29 @@ if form.getvalue('action_waf') is not None and serv is not None: funct.is_restarted(serv, action) - funct.logging(serv, 'WAF service has been ' + action + 'ed', haproxywi=1, login=1, keep_history=1, service='haproxy') + funct.logging(serv, 'HAProxy WAF service has been ' + action + 'ed', haproxywi=1, login=1, keep_history=1, service='haproxy') commands = ["sudo systemctl %s waf" % action] funct.ssh_command(serv, commands) +if form.getvalue('action_waf_nginx') is not None and serv is not None: + serv = form.getvalue('serv') + action = form.getvalue('action_waf_nginx') + config_dir = funct.return_nice_path(sql.get_setting('nginx_dir')) + + if action not in ('start', 'stop'): + print('error: wrong action') + sys.exit() + + funct.is_restarted(serv, action) + + waf_new_state = 'on' if action == 'start' else 'off' + waf_old_state = 'off' if action == 'start' else 'on' + + funct.logging(serv, 'NGINX WAF service has been ' + action + 'ed', haproxywi=1, login=1, keep_history=1, service='nginx') + commands = [ f"sudo sed -i 's/modsecurity {waf_old_state}/modsecurity {waf_new_state}/g' {config_dir}nginx.conf" + f" && sudo systemctl reload nginx" ] + funct.ssh_command(serv, commands) + if form.getvalue('action_apache') is not None and serv is not None: action = form.getvalue('action_apache') @@ -728,32 +750,44 @@ if act == "overviewwaf": ) template = env.get_template('overivewWaf.html') + waf_service = form.getvalue('service') servers = sql.select_servers(server=serv) cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE")) user_id = cookie.get('uuid') - haproxy_path = '' + config_path = '' returned_servers = [] waf = '' metrics_en = 0 waf_process = '' waf_mode = '' + is_waf_on_server = 0 for server in servers: - haproxy = sql.select_haproxy(server[2]) - if haproxy == 1: - haproxy_path = sql.get_setting('haproxy_dir') - waf = sql.select_waf_servers(server[2]) - metrics_en = sql.select_waf_metrics_enable_server(server[2]) + if waf_service == 'haproxy': + is_waf_on_server = sql.select_haproxy(server[2]) + elif waf_service == 'nginx': + is_waf_on_server = sql.select_nginx(server[2]) + + if is_waf_on_server == 1: + config_path = sql.get_setting(waf_service + '_dir') + if waf_service == 'haproxy': + waf = sql.select_waf_servers(server[2]) + metrics_en = sql.select_waf_metrics_enable_server(server[2]) + elif waf_service == 'nginx': + waf = sql.select_waf_nginx_servers(server[2]) try: waf_len = len(waf) except Exception: waf_len = 0 if waf_len >= 1: - command = ["ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l"] + if waf_service == 'haproxy': + command = ["ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l"] + elif waf_service == 'nginx': + command = ["grep 'modsecurity on' %s* --exclude-dir=waf -Rs |wc -l" % funct.return_nice_path(config_path)] commands1 = [ - "grep SecRuleEngine %s/waf/modsecurity.conf |grep -v '#' |awk '{print $2}'" % haproxy_path] + "grep SecRuleEngine %s/waf/modsecurity.conf |grep -v '#' |awk '{print $2}'" % config_path] waf_process = funct.ssh_command(server[2], command) waf_mode = funct.ssh_command(server[2], commands1).strip() @@ -774,7 +808,7 @@ if act == "overviewwaf": returned_servers.append(server_status) servers_sorted = sorted(returned_servers, key=funct.get_key) - template = template.render(service_status=servers_sorted, role=sql.get_user_role_by_uuid(user_id.value)) + template = template.render(service_status=servers_sorted, role=sql.get_user_role_by_uuid(user_id.value), waf_service=waf_service) print(template) if act == "overviewServers": @@ -1836,7 +1870,11 @@ if form.getvalue('haproxyaddserv'): hapver=form.getvalue('hapver'), docker=form.getvalue('docker')) if form.getvalue('installwaf'): - funct.waf_install(form.getvalue('installwaf')) + service = form.getvalue('service') + if service == 'haproxy': + funct.waf_install(form.getvalue('installwaf')) + else: + funct.waf_nginx_install(form.getvalue('installwaf')) if form.getvalue('update_roxy_wi'): service = form.getvalue('service') @@ -2150,8 +2188,8 @@ if form.getvalue('get_lists'): lib_path = funct.get_config_var('main', 'lib_path') list_path = lib_path + "/" + sql.get_setting('lists_path') + "/" + form.getvalue('group') + "/" + form.getvalue('color') lists = funct.get_files(dir=list_path, format="lst") - for list in lists: - print(list) + for l in lists: + print(l) if form.getvalue('get_ldap_email'): username = form.getvalue('get_ldap_email') @@ -2191,11 +2229,17 @@ if form.getvalue('get_ldap_email'): ldap_bind.unbind() if form.getvalue('change_waf_mode'): - waf_mode = form.getvalue('change_waf_mode') + waf_mode = funct.checkAjaxInput(form.getvalue('change_waf_mode')) server_hostname = form.getvalue('server_hostname') - haproxy_dir = sql.get_setting('haproxy_dir') + service = funct.checkAjaxInput(form.getvalue('service')) serv = sql.select_server_by_name(server_hostname) - commands = ["sudo sed -i 's/^SecRuleEngine.*/SecRuleEngine %s/' %s/waf/modsecurity.conf " % (waf_mode, haproxy_dir)] + + if service == 'haproxy': + config_dir = sql.get_setting('haproxy_dir') + elif service == 'nginx': + config_dir = sql.get_setting('nginx_dir') + + commands = ["sudo sed -i 's/^SecRuleEngine.*/SecRuleEngine %s/' %s/waf/modsecurity.conf " % (waf_mode, config_dir)] funct.ssh_command(serv, commands) funct.logging(serv, 'Has been changed WAF mod to ' + waf_mode, haproxywi=1, login=1) @@ -2259,6 +2303,8 @@ if form.getvalue('updateuser') is not None: if form.getvalue('updatepassowrd') is not None: password = form.getvalue('updatepassowrd') + username = '' + if form.getvalue('uuid'): user_id = sql.get_user_id_by_uuid(form.getvalue('uuid')) else: @@ -2478,6 +2524,9 @@ if form.getvalue('new_ssh'): if form.getvalue('sshdel') is not None: lib_path = funct.get_config_var('main', 'lib_path') sshdel = funct.checkAjaxInput(form.getvalue('sshdel')) + name = '' + ssh_enable = 0 + ssh_key_name = '' for sshs in sql.select_ssh(id=sshdel): ssh_enable = sshs.enable @@ -2501,6 +2550,7 @@ if form.getvalue('updatessh'): group = form.getvalue('group') username = form.getvalue('ssh_user') password = form.getvalue('ssh_pass') + new_ssh_key_name = '' if username is None: print(error_mess) @@ -2553,10 +2603,6 @@ if form.getvalue('ssh_cert'): ssh_keys = full_dir + name + '.pem' try: - # cloud = sql.is_cloud() - # if cloud != '': - # key.write_private_key_file(ssh_keys, password=cloud) - # else: key.write_private_key_file(ssh_keys) except Exception as e: print('error: Cannot save SSH key file: ', str(e)) @@ -2829,17 +2875,17 @@ if form.getvalue('showBytes') is not None: port = sql.get_setting('haproxy_sock_port') bin_bout = [] cmd = "echo 'show stat' |nc {} {} |cut -d ',' -f 1-2,9|grep -E '[0-9]'|awk -F',' '{{sum+=$3;}}END{{print sum;}}'".format(serv, port) - bin, stderr = funct.subprocess_execute(cmd) - bin_bout.append(bin[0]) + bit_in, stderr = funct.subprocess_execute(cmd) + bin_bout.append(bit_in[0]) cmd = "echo 'show stat' |nc {} {} |cut -d ',' -f 1-2,10|grep -E '[0-9]'|awk -F',' '{{sum+=$3;}}END{{print sum;}}'".format(serv, port) - bin, stderr = funct.subprocess_execute(cmd) - bin_bout.append(bin[0]) + bout, stderr = funct.subprocess_execute(cmd) + bin_bout.append(bout[0]) cmd = "echo 'show stat' |nc {} {} |cut -d ',' -f 1-2,5|grep -E '[0-9]'|awk -F',' '{{sum+=$3;}}END{{print sum;}}'".format(serv, port) - bin, stderr = funct.subprocess_execute(cmd) - bin_bout.append(bin[0]) + cin, stderr = funct.subprocess_execute(cmd) + bin_bout.append(cin[0]) cmd = "echo 'show stat' |nc {} {} |cut -d ',' -f 1-2,8|grep -E '[0-9]'|awk -F',' '{{sum+=$3;}}END{{print sum;}}'".format(serv, port) - bin, stderr = funct.subprocess_execute(cmd) - bin_bout.append(bin[0]) + cout, stderr = funct.subprocess_execute(cmd) + bin_bout.append(cout[0]) from jinja2 import Environment, FileSystemLoader @@ -2883,7 +2929,8 @@ if form.getvalue('waf_rule_id'): haproxy_path = sql.get_setting('haproxy_dir') rule_file = sql.select_waf_rule_by_id(rule_id) conf_file_path = haproxy_path + '/waf/modsecurity.conf' - rule_file_path = 'Include ' + haproxy_path + '//waf/rules/' + rule_file + rule_file_path = 'Include ' + haproxy_path + '/waf/rules/' + rule_file + print(rule_file_path) if enable == '0': cmd = ["sudo sed -i 's!" + rule_file_path + "!#" + rule_file_path + "!' " + conf_file_path] @@ -2901,6 +2948,32 @@ if form.getvalue('waf_rule_id'): print(funct.ssh_command(serv, cmd)) sql.update_enable_waf_rules(rule_id, serv, enable) +if form.getvalue('new_waf_rule'): + service = form.getvalue('service') + new_waf_rule = form.getvalue('new_waf_rule') + new_rule_desc = form.getvalue('new_rule_description') + rule_file = form.getvalue('new_rule_file') + rule_file = rule_file + '.conf' + waf_path = '' + + if service == 'haproxy': + waf_path = funct.return_nice_path(sql.get_setting('haproxy_dir')) + elif service == 'nginx': + waf_path = funct.return_nice_path(sql.get_setting('nginx_dir')) + + conf_file_path = waf_path + 'waf/modsecurity.conf' + rule_file_path = waf_path + 'waf/rules/' + rule_file + + cmd = [f"sudo echo Include {rule_file_path} >> {conf_file_path} && sudo touch {rule_file_path}"] + print(funct.ssh_command(serv, cmd)) + print(sql.insert_new_waf_rule(new_waf_rule, rule_file, new_rule_desc, service, serv)) + + try: + funct.logging('WAF', ' A new rule has been created ' + rule_file + ' on the server ' + serv, + haproxywi=1, login=1) + except Exception: + pass + if form.getvalue('lets_domain'): serv = form.getvalue('serv') lets_domain = form.getvalue('lets_domain') @@ -3251,7 +3324,7 @@ if form.getvalue('show_versions'): if form.getvalue('get_group_name_by_id'): print(sql.get_group_name_by_id(form.getvalue('get_group_name_by_id'))) -if form.getvalue('do_new_name') or form.getvalue('aws_new_name') or form.getvalue('gcore_new_name'): +if any((form.getvalue('do_new_name'), form.getvalue('aws_new_name'), form.getvalue('gcore_new_name'))): funct.check_user_group() is_add = False if form.getvalue('do_new_name'): @@ -3288,6 +3361,8 @@ if form.getvalue('do_new_name') or form.getvalue('aws_new_name') or form.getvalu cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE")) user_uuid = cookie.get('uuid') role_id = sql.get_user_role_by_uuid(user_uuid.value) + params = sql.select_provisioning_params() + providers = sql.select_providers(provider_group, key=provider_token) if role_id == 1: groups = sql.select_groups() @@ -3296,7 +3371,7 @@ if form.getvalue('do_new_name') or form.getvalue('aws_new_name') or form.getvalu env = Environment(loader=FileSystemLoader('templates'), autoescape=True) template = env.get_template('ajax/provisioning/providers.html') - template = template.render(providers=sql.select_providers(provider_group, key=provider_token), role=role_id, groups=groups, user_group=provider_group, adding=1) + template = template.render(providers=providers, role=role_id, groups=groups, user_group=provider_group, adding=1, params=params) print(template) if form.getvalue('providerdel'): @@ -3462,12 +3537,13 @@ if form.getvalue('doworkspace'): user, user_id, role, token, servers, user_services = funct.get_users_params() new_server = sql.select_provisioned_servers(new=workspace, group=group, type='do') + params = sql.select_provisioning_params() env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/provisioned_servers.html') template = template.render( servers=new_server, groups=sql.select_groups(), user_group=group, - providers=sql.select_providers(group), role=role, adding=1 + providers=sql.select_providers(group), role=role, adding=1, params=params ) print(template) @@ -3557,12 +3633,13 @@ if form.getvalue('awsworkspace'): user, user_id, role, token, servers, user_services = funct.get_users_params() new_server = sql.select_provisioned_servers(new=workspace, group=group, type='aws') + params = sql.select_provisioning_params() env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/provisioned_servers.html') template = template.render( servers=new_server, groups=sql.select_groups(), user_group=group, - providers=sql.select_providers(group), role=role, adding=1 + providers=sql.select_providers(group), role=role, adding=1, params=params ) print(template) @@ -3682,6 +3759,10 @@ if ( for ip in output: ips += ip ips += ' ' + + if cloud == 'gcore': + ips = ips.split(' ')[0] + print(ips) sql.update_provisioning_server_status('Created', group, workspace, provider_id, update_ip=ips) @@ -3826,6 +3907,7 @@ if form.getvalue('gcoreworkspace'): user, user_id, role, token, servers, user_services = funct.get_users_params() new_server = sql.select_provisioned_servers(new=workspace, group=group, type='gcore') + params = sql.select_provisioning_params() env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/provisioned_servers.html') @@ -3834,7 +3916,8 @@ if form.getvalue('gcoreworkspace'): user_group=group, providers=sql.select_providers(group), role=role, - adding=1) + adding=1, + params=params) print(template) if form.getvalue('gcoreeditworkspace'): @@ -3879,33 +3962,42 @@ if form.getvalue('editAwsServer'): funct.check_user_group() server_id = form.getvalue('editAwsServer') user_group = form.getvalue('editAwsGroup') + params = sql.select_provisioning_params() + providers = sql.select_providers(int(user_group)) + server = sql.select_gcore_server(server_id=server_id) from jinja2 import Environment, FileSystemLoader env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/aws_edit_dialog.html') - template = template.render(server=sql.select_aws_server(server_id=server_id), providers=sql.select_providers(int(user_group))) + template = template.render(server=server, providers=providers, params=params) print(template) if form.getvalue('editGcoreServer'): funct.check_user_group() server_id = form.getvalue('editGcoreServer') user_group = form.getvalue('editGcoreGroup') + params = sql.select_provisioning_params() + providers = sql.select_providers(int(user_group)) + server = sql.select_gcore_server(server_id=server_id) from jinja2 import Environment, FileSystemLoader env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/gcore_edit_dialog.html') - template = template.render(server=sql.select_gcore_server(server_id=server_id), providers=sql.select_providers(int(user_group))) + template = template.render(server=server, providers=providers, params=params) print(template) if form.getvalue('editDoServer'): funct.check_user_group() server_id = form.getvalue('editDoServer') user_group = form.getvalue('editDoGroup') + params = sql.select_provisioning_params() + providers = sql.select_providers(int(user_group)) + server = sql.select_do_server(server_id=server_id) from jinja2 import Environment, FileSystemLoader env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader('templates')) template = env.get_template('ajax/provisioning/do_edit_dialog.html') - template = template.render(server=sql.select_do_server(server_id=server_id), providers=sql.select_providers(int(user_group))) + template = template.render(server=server, providers=providers, params=params) print(template) if form.getvalue('edit_do_provider'): @@ -4009,12 +4101,12 @@ if form.getvalue('load_update_hapwi'): template = env.get_template('ajax/load_updatehapwi.html') versions = funct.versions() - checker_ver = funct.check_new_version(service='checker') - smon_ver = funct.check_new_version(service='smon') - metrics_ver = funct.check_new_version(service='metrics') - keep_ver = funct.check_new_version(service='keep') - portscanner_ver = funct.check_new_version(service='portscanner') - socket_ver = funct.check_new_version(service='socket') + checker_ver = funct.check_new_version('checker') + smon_ver = funct.check_new_version('smon') + metrics_ver = funct.check_new_version('metrics') + keep_ver = funct.check_new_version('keep_alive') + portscanner_ver = funct.check_new_version('portscanner') + socket_ver = funct.check_new_version('socket') services = funct.get_services_status() template = template.render(services=services, diff --git a/app/scripts/ansible/roles/waf/tasks/main.yml b/app/scripts/ansible/roles/waf/tasks/main.yml index 01130af4..a62e0bbe 100644 --- a/app/scripts/ansible/roles/waf/tasks/main.yml +++ b/app/scripts/ansible/roles/waf/tasks/main.yml @@ -15,6 +15,18 @@ msg="info HAProxy WAF has already installed" when: stat_result.stat.exists + - name: erase the RPMS for HAProxy + yum: + name: + - ssdeep + - ssdeep-devel + state: absent + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" + - name: install the el7 RPMS for HAProxy yum: name: @@ -65,6 +77,7 @@ - libevent-devel - libtool - make + - gcc-c++ state: latest when: - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' @@ -86,6 +99,7 @@ - libyajl-dev - libxml2 - automake + - g++ - make state: present when: ansible_facts['os_family'] == 'Debian' or ansible_facts['os_family'] == 'Ubuntu' @@ -110,57 +124,41 @@ become: false unarchive: src: /tmp/modsecurity.tar.gz - dest: /tmp/modsecurity/ + dest: /tmp/modsecurity remote_src: true - - name: Copy modsecurity - copy: - src: "/tmp/modsecurity/modsecurity-{{ modsec_ver }}/" - dest: /tmp/modsecurity/ - remote_src: yes - - - name: Set execute permision to configure - become: true - command: chdir=/tmp/modsecurity/ chmod +x configure - args: - warn: no + - name: Set ModSec src foleder + set_fact: + mod_sec_src: /tmp/modsecurity-{{ modsec_ver }} - name: Re configure Modsecurity become: true - command: chdir=/tmp/modsecurity/ autoreconf -f -i + command: "chdir={{ mod_sec_src }} autoreconf -f -i" - name: Configure Modsecurity become: true - command: chdir=/tmp/modsecurity/ ./configure --prefix=/tmp/modsecurity --enable-standalone-module --disable-mlogc --enable-pcre-study --without-lua --enable-pcre-jit + command: "chdir={{ mod_sec_src }} ./configure --prefix=/tmp/modsecurity --enable-standalone-module --disable-mlogc --enable-pcre-study --without-lua --enable-pcre-jit" - name: Make Modsecurity - command: chdir=/tmp/modsecurity/ make + command: "chdir={{ mod_sec_src }} make" - name: Make Install Modsecurity - command: chdir=/tmp/modsecurity/ make -C standalone install + command: "chdir={{ mod_sec_src }} make -C standalone install" - name: Creates directory file: - path: /tmp/modsecurity/INSTALL/include + path: "{{ mod_sec_src }}INSTALL/include" state: directory - name: Copy Modsec libs copy: - src: /tmp/modsecurity/standalone/.libs/ - dest: /tmp/modsecurity/INSTALL/include/ - remote_src: yes - - - name: Copy Modsec files - copy: - src: /tmp/modsecurity/standalone/ - dest: /tmp/modsecurity/INSTALL/include/ - remote_src: yes - - - name: Copy Modsec apache files - copy: - src: /tmp/modsecurity/apache2/ - dest: /tmp/modsecurity/INSTALL/include/ + src: "{{ mod_sec_src }}/{{ item }}" + dest: "{{ mod_sec_src }}/INSTALL/include/" remote_src: yes + with_items: + - standalone/.libs/ + - standalone/ + - apache2/ - name: Install git package: @@ -175,22 +173,20 @@ mod_sec_dir: /tmp/spoa-modsecurity - name: Make APT Modsecurity module for HAProxy - command: "chdir={{ mod_sec_dir }} make MODSEC_INC=/tmp/modsecurity/INSTALL/include MODSEC_LIB=/tmp/modsecurity/INSTALL/include APACHE2_INC=/usr/include/apache2/ APR_INC=/usr/include/apr-1.0" + command: "chdir={{ mod_sec_dir }} make MODSEC_INC={{ mod_sec_src }}/INSTALL/include MODSEC_LIB={{ mod_sec_src }}/INSTALL/include APACHE2_INC=/usr/include/apache2/ APR_INC=/usr/include/apr-1.0" when: ansible_facts['os_family'] == 'Debian' or ansible_facts['os_family'] == 'Ubuntu' - name: Make EL Modsecurity module for HAProxy - command: "chdir={{ mod_sec_dir }} make MODSEC_INC=/tmp/modsecurity/INSTALL/include MODSEC_LIB=/tmp/modsecurity/INSTALL/include APACHE2_INC=/usr/include/httpd/ APR_INC=/usr/include/apr-1" + command: "chdir={{ mod_sec_dir }} make MODSEC_INC={{ mod_sec_src }}/INSTALL/include MODSEC_LIB={{ mod_sec_src }}/INSTALL/include APACHE2_INC=/usr/include/httpd/ APR_INC=/usr/include/apr-1" when: ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' - name: Make WAF rules directory file: - path: "{{ HAPROXY_PATH }}/waf/rules" - state: directory - - - name: Make WAF bin directory - file: - path: "{{ HAPROXY_PATH }}/waf/bin" + path: "{{ HAPROXY_PATH }}/waf/{{ item }}" state: directory + with_items: + - rules + - bin - name: Copy Modsec module to HAProxy dir copy: @@ -269,9 +265,16 @@ - name: Copy owasp files copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/owasp-modsecurity-crs-2.2.9/ + src: "/tmp/owasp-modsecurity-crs-2.2.9/{{ item }}" dest: /tmp/owasp-modsecurity-crs-2.2.9 remote_src: yes + with_items: + - owasp-modsecurity-crs-2.2.9/ + - activated_rules/ + - base_rules/ + - experimental_rules/ + - optional_rules/ + - slr_rules/ - name: Copy Modsec crs conf file copy: @@ -279,36 +282,6 @@ dest: "{{ HAPROXY_PATH }}/waf/rules/modsecurity_crs_10_setup.conf" remote_src: true - - name: Copy Modsec crs activated_rules files - copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/activated_rules/ - dest: "{{ HAPROXY_PATH }}/waf/rules/" - remote_src: yes - - - name: Copy Modsec crs base_rules files - copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/base_rules/ - dest: "{{ HAPROXY_PATH }}/waf/rules/" - remote_src: yes - - - name: Copy Modsec crs experimental_rules files - copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/experimental_rules/ - dest: "{{ HAPROXY_PATH }}/waf/rules/" - remote_src: yes - - - name: Copy Modsec crs optional_rules files - copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/optional_rules/ - dest: "{{ HAPROXY_PATH }}/waf/rules/" - remote_src: yes - - - name: Copy Modsec crs slr_rules files - copy: - src: /tmp/owasp-modsecurity-crs-2.2.9/slr_rules/ - dest: "{{ HAPROXY_PATH }}/waf/rules/" - remote_src: yes - - name: Ensure ModSec engine mode on ansible.builtin.lineinfile: path: "{{ HAPROXY_PATH }}/waf/modsecurity.conf" @@ -367,42 +340,14 @@ enabled: yes always: - - name: Remove modsecurity.tar.gz + - name: Clean up ansible.builtin.file: - path: /tmp/modsecurity.tar.gz - state: absent - - - name: Remove modsecurity-2.9.2 - ansible.builtin.file: - path: /tmp/modsecurity-2.9.2 - state: absent - - - name: Remove HAProxy - ansible.builtin.file: - path: "/tmp/haproxy-{{ VERSION }}" - state: absent - - - name: Remove modsecurity - ansible.builtin.file: - path: /tmp/modsecurity - state: absent - - - name: Remove modsecurity.conf - ansible.builtin.file: - path: /tmp/modsecurity.conf - state: absent - - - name: Remove owasp.tar.gz - ansible.builtin.file: - path: /tmp/owasp.tar.gz - state: absent - - - name: Remove owasp-modsecurity-crs-2.2.9 - ansible.builtin.file: - path: /tmp/owasp-modsecurity-crs-2.2.9 - state: absent - - - name: Remove spoa-modsecurity - ansible.builtin.file: - path: /tmp/spoa-modsecurity + path: "{{ item }}" state: absent + with_items: + - /tmp/modsecurity.tar.gz + - "/tmp/modsecurity-{{ modsec_ver }}" + - "/tmp/haproxy-{{ VERSION }}" + - /tmp/owasp.tar.gz + - /tmp/owasp-modsecurity-crs-2.2.9 + - /tmp/spoa-modsecurity diff --git a/app/scripts/ansible/roles/waf_nginx.yml b/app/scripts/ansible/roles/waf_nginx.yml new file mode 100644 index 00000000..b99fa788 --- /dev/null +++ b/app/scripts/ansible/roles/waf_nginx.yml @@ -0,0 +1,11 @@ +--- +- name: Install WAF + hosts: "{{ variable_host }}" + become: yes + become_method: sudo + gather_facts: yes + roles: + - role: waf_nginx + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" diff --git a/app/scripts/ansible/roles/waf_nginx/defaults/main.yml b/app/scripts/ansible/roles/waf_nginx/defaults/main.yml new file mode 100644 index 00000000..a28b5cab --- /dev/null +++ b/app/scripts/ansible/roles/waf_nginx/defaults/main.yml @@ -0,0 +1 @@ +coreruleset_ver: 3.3.2 \ No newline at end of file diff --git a/app/scripts/ansible/roles/waf_nginx/handlers/main.yml b/app/scripts/ansible/roles/waf_nginx/handlers/main.yml new file mode 100644 index 00000000..f71b04a0 --- /dev/null +++ b/app/scripts/ansible/roles/waf_nginx/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reload NGINX + service: name=nginx state=reloaded diff --git a/app/scripts/ansible/roles/waf_nginx/tasks/main.yml b/app/scripts/ansible/roles/waf_nginx/tasks/main.yml new file mode 100644 index 00000000..53afd46c --- /dev/null +++ b/app/scripts/ansible/roles/waf_nginx/tasks/main.yml @@ -0,0 +1,224 @@ +--- +- name: Installing WAF + block: + - name: Set SSH port + set_fact: + ansible_port: "{{SSH_PORT}}" + +# - debug: msg="{{ ansible_facts }}" + + - name: Check that WAF has been installed + stat: + path: "{{ NGINX_PATH }}/waf/modsecurity.conf" + register: stat_result + + - name: Fail if has been installed + fail: + msg="info NGINX WAF has already installed" + when: stat_result.stat.exists + + - name: install the common RPMS for NGINX + yum: + name: + - libtool + - libxml2-devel + - gcc + - curl-devel + - pcre-devel + - automake + - autoconf + - libevent-devel + - libtool + - make + - gcc-c++ + - git + - redhat-rpm-config + - openssl-devel + - libxslt-devel + - gd-devel + - perl-ExtUtils-Embed + - GeoIP-devel + - ssdeep-devel + state: latest + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" + + - name: Install needed packages + apt: + name: + - libtool + - libevent-dev + - libpcre3-dev + - libxml2-dev + - gcc + - libpcre3-dev + - libcurl4-nss-dev + - libyajl-dev + - libxml2 + - automake + - autoconf + - g++ + - make + - openssl-dev + - libxslt-dev + - gd-dev + - perl-modules + - libmodsecurity3 + - libgeoip-dev + - libfuzzy2 + state: present + when: ansible_facts['os_family'] == 'Debian' or ansible_facts['os_family'] == 'Ubuntu' + environment: + http_proxy: "{{PROXY}}" + https_proxy: "{{PROXY}}" + + - name: Download ModSec + ansible.builtin.get_url: + url: "http://repo.roxy-wi.org/modsec/modsecv3.0.7-{{ ansible_facts.distribution | lower }}{{ ansible_facts.distribution_major_version }}.tar.gz" + dest: /usr/local/modsecv3.tar.gz + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + + - name: Untar ModSec + unarchive: + src: /usr/local/modsecv3.tar.gz + dest: /usr/local/ + remote_src: true + + - name: Get NGINX version + shell: /usr/sbin/nginx -v + register: nginx_version + + - name: Get NGINX parameters + shell: /usr/sbin/nginx -V 2>&1 |grep configu |awk -F":" '{print $2}' + register: nginx_params + + - name: Clone NGINX connector + shell: git clone --depth 1 https://github.com/SpiderLabs/ModSecurity-nginx.git /tmp/nginx-connector + + - name: Download NGINX + ansible.builtin.get_url: + url: "http://nginx.org/download/nginx-{{ nginx_version.stderr.split('/')[1] }}.tar.gz" + dest: /tmp/nginx_src.tar.gz + + - name: Create nginx_src directory + become: false + file: + path: /tmp/nginx_src/ + state: directory + + - name: Untar NGINX + become: false + unarchive: + src: /tmp/nginx_src.tar.gz + dest: /tmp/nginx_src/ + remote_src: true + + - name: Configure NGINX + become: true + command: "chdir=/tmp/nginx_src/nginx-{{ nginx_version.stderr.split('/')[1] }} ./configure {{ nginx_params.stdout }} --add-dynamic-module=../../nginx-connector" + environment: + CFLAGS: -Wno-error + + - name: Make NGINX modules + become: true + command: "chdir=/tmp/nginx_src/nginx-{{ nginx_version.stderr.split('/')[1] }} make modules" + + - name: Copy module for CentOS + become: true + command: "chdir=/tmp/nginx_src/nginx-{{ nginx_version.stderr.split('/')[1] }} cp objs/ngx_http_modsecurity_module.so /usr/share/nginx/modules/" + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + + - name: Copy module for Ubuntu + become: true + command: "chdir=/tmp/nginx_src/nginx-{{ nginx_version.stderr.split('/')[1] }} cp objs/ngx_http_modsecurity_module.so /usr/lib/nginx/modules/" + when: ansible_facts['os_family'] == 'Debian' or ansible_facts['os_family'] == 'Ubuntu' + + - name: Enable module for Centos + become: true + shell: echo 'load_module "modules/ngx_http_modsecurity_module.so";' > /usr/share/nginx/modules/mod-waf-connector.conf + when: + - ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS' + + - name: Enable module for Ubuntu + lineinfile: + path: "{{ NGINX_PATH }}/nginx.conf" + line: load_module modules/ngx_http_modsecurity_module.so; + insertbefore: BOF + when: ansible_facts['os_family'] == 'Debian' or ansible_facts['os_family'] == 'Ubuntu' + + - name: Create WAF directory + become: false + file: + path: "{{ NGINX_PATH }}/waf/" + state: directory + + - name: Create WAF rules directory + become: false + file: + path: "{{ NGINX_PATH }}/waf/rules" + state: directory + + - name: Download modsecurity.conf + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/SpiderLabs/ModSecurity/v3/master/modsecurity.conf-recommended + dest: "{{ NGINX_PATH }}/waf/modsecurity.conf" + + - name: Download unicode.mapping + ansible.builtin.get_url: + url: https://github.com/SpiderLabs/ModSecurity/blob/v3/master/unicode.mapping + dest: "{{ NGINX_PATH }}/waf/unicode.mapping" + + - name: Create WAF config + template: + src: waf.conf.j2 + dest: "{{ NGINX_PATH }}/waf/waf.conf" + + - name: Download OWASP rules + ansible.builtin.get_url: + url: "https://github.com/coreruleset/coreruleset/archive/v{{ coreruleset_ver }}.tar.gz" + dest: /tmp/OWASP.tar.gz + + - name: Untar NGINX + become: false + unarchive: + src: /tmp/OWASP.tar.gz + dest: /tmp/ + remote_src: true + + - name: Copy Modsec crs activated_rules files + copy: + src: "/tmp/coreruleset-{{ coreruleset_ver }}/rules/" + dest: "{{ NGINX_PATH }}/waf/rules/" + remote_src: yes + + - name: Copy module + become: true + command: "chdir=/tmp/coreruleset-{{ coreruleset_ver }} cp crs-setup.conf.example {{ NGINX_PATH }}/waf/rulescrs-setup.conf" + + - name: Add waf Mod on + ansible.builtin.blockinfile: + path: "{{ NGINX_PATH }}/nginx.conf" + marker: "#-- {mark} WAF BLOCK --#" + insertafter: "http {" + block: | + modsecurity off; + modsecurity_rules_file /etc/nginx/waf/waf.conf; + notify: reload NGINX + + always: + - name: Clean up + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: + - /tmp/nginx_src/ + - /tmp/nginx_src.tar.gz + - /tmp/nginx-connector + - /tmp/OWASP.tar.gz + - /usr/local/modsecv3.tar.gz +# - "/tmp/coreruleset-{{ coreruleset_ver }}" diff --git a/app/scripts/ansible/roles/waf_nginx/templates/waf.conf.j2 b/app/scripts/ansible/roles/waf_nginx/templates/waf.conf.j2 new file mode 100644 index 00000000..88be7bee --- /dev/null +++ b/app/scripts/ansible/roles/waf_nginx/templates/waf.conf.j2 @@ -0,0 +1,31 @@ +Include {{ NGINX_PATH }}/waf/modsecurity.conf +Include {{ NGINX_PATH }}/waf/rulescrs-setup.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-901-INITIALIZATION.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-903.9001-DRUPAL-EXCLUSION-RULES.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-903.9003-NEXTCLOUD-EXCLUSION-RULES.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-903.9004-DOKUWIKI-EXCLUSION-RULES.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-903.9005-CPANEL-EXCLUSION-RULES.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-903.9006-XENFORO-EXCLUSION-RULES.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-905-COMMON-EXCEPTIONS.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-910-IP-REPUTATION.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-911-METHOD-ENFORCEMENT.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-912-DOS-PROTECTION.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-913-SCANNER-DETECTION.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-921-PROTOCOL-ATTACK.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-931-APPLICATION-ATTACK-RFI.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-932-APPLICATION-ATTACK-RCE.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-933-APPLICATION-ATTACK-PHP.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-934-APPLICATION-ATTACK-NODEJS.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-943-APPLICATION-ATTACK-SESSION-FIXATION.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-944-APPLICATION-ATTACK-JAVA.conf +Include {{ NGINX_PATH }}/waf/rules/REQUEST-949-BLOCKING-EVALUATION.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-951-DATA-LEAKAGES-SQL.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-952-DATA-LEAKAGES-JAVA.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-953-DATA-LEAKAGES-PHP.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-954-DATA-LEAKAGES-IIS.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-959-BLOCKING-EVALUATION.conf +Include {{ NGINX_PATH }}/waf/rules/RESPONSE-980-CORRELATION.conf diff --git a/app/scripts/waf_nginx.sh b/app/scripts/waf_nginx.sh new file mode 100644 index 00000000..451e8d97 --- /dev/null +++ b/app/scripts/waf_nginx.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +for ARGUMENT in "$@" +do + KEY=$(echo "$ARGUMENT" | cut -f1 -d=) + VALUE=$(echo "$ARGUMENT" | cut -f2 -d=) + + case "$KEY" in + PROXY) PROXY=${VALUE} ;; + NGINX_PATH) NGINX_PATH=${VALUE} ;; + HOST) HOST=${VALUE} ;; + USER) USER=${VALUE} ;; + PASS) PASS=${VALUE} ;; + KEY) KEY=${VALUE} ;; + SSH_PORT) SSH_PORT=${VALUE} ;; + *) + esac +done + + +export ANSIBLE_HOST_KEY_CHECKING=False +export ANSIBLE_DISPLAY_SKIPPED_HOSTS=False +export ACTION_WARNINGS=False +export LOCALHOST_WARNING=False +export COMMAND_WARNINGS=False + +PWD=$(pwd) +PWD=$PWD/scripts/ansible/ +echo "$HOST ansible_port=$SSH_PORT" > $PWD/$HOST + +if [[ $KEY == "" ]]; then + ansible-playbook $PWD/roles/waf_nginx.yml -e "ansible_user=$USER ansible_ssh_pass='$PASS' variable_host=$HOST PROXY=$PROXY NGINX_PATH=$NGINX_PATH SSH_PORT=$SSH_PORT" -i $PWD/$HOST +else + ansible-playbook $PWD/roles/waf_nginx.yml --key-file $KEY -e "ansible_user=$USER variable_host=$HOST PROXY=$PROXY NGINX_PATH=$NGINX_PATH SSH_PORT=$SSH_PORT" -i $PWD/$HOST +fi + +if [ $? -gt 0 ] +then + echo "error: Cannot install WAF" +else + echo "success" +fi +rm -f $PWD/$HOST diff --git a/app/sql.py b/app/sql.py index c09c18d0..61cfe3f6 100755 --- a/app/sql.py +++ b/app/sql.py @@ -1272,6 +1272,25 @@ def select_waf_servers(serv): return en.ip +def select_waf_nginx_servers(serv): + query = Server.select(Server.ip).join(WafNginx, on=(WafNginx.server_id == Server.server_id)).where(Server.ip == serv) + try: + query_res = query.execute() + except Exception as e: + out_error(e) + else: + for en in query_res: + return en.ip + + +def insert_waf_nginx_server(server_ip): + try: + server_id = Server.get(Server.ip == server_ip).server_id + WafNginx.insert(server_id=server_id).execute() + except Exception as e: + out_error(e) + + def select_waf_servers_metrics_for_master(): query = Server.select(Server.ip).join( Waf, on=(Waf.server_id == Server.server_id) @@ -1390,8 +1409,105 @@ def insert_waf_rules(serv): return True -def select_waf_rules(serv): - query = WafRules.select(WafRules.id, WafRules.rule_name, WafRules.en, WafRules.desc).where(WafRules.serv == serv) +def insert_nginx_waf_rules(serv): + data_source = [ + {'serv': serv, 'rule_name': 'Initialization', 'rule_file': 'REQUEST-901-INITIALIZATION.conf', + 'desc': 'This file REQUEST-901-INITIALIZATION.conf initializes the Core Rules and performs preparatory actions. ' + 'It also fixes errors and omissions of variable definitions in the file crs-setup.conf The setup.conf' + 'can and should be edited by the user, this file. is part of the CRS installation and should not be altered.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Drupal exclusion rules', 'rule_file': 'REQUEST-903.9001-DRUPAL-EXCLUSION-RULES.conf', + 'desc': 'These exclusions remedy false positives in a default Drupal install. The exclusions are only active ' + 'if crs_exclusions_drupal=1 is set. See rule 900130 in crs-setup.conf for instructions.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Nextcloud exclusion rules', 'rule_file': 'REQUEST-903.9003-NEXTCLOUD-EXCLUSION-RULES.conf', + 'desc': 'These exclusions remedy false positives in a default NextCloud install. They will likely work with OwnCloud ' + 'too, but you may have to modify them. The exclusions are only active if crs_exclusions_nextcloud=1 is set. ' + 'See rule 900130 in crs-setup.conf for instructions.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Dokuwiki exclusion rules', 'rule_file': 'REQUEST-903.9004-DOKUWIKI-EXCLUSION-RULES.conf', + 'desc': 'These exclusions remedy false positives in a default Dokuwiki install. The exclusions are only active ' + 'if crs_exclusions_dokuwiki=1 is set. See rule 900130 in crs-setup.conf for instructions.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'CPanel exclusion rules', 'rule_file': 'REQUEST-903.9005-CPANEL-EXCLUSION-RULES.conf', + 'desc': 'These exclusions remedy false positives in a default CPanel install. The exclusions are only active ' + 'if crs_exclusions_cpanel=1 is set. See rule 900130 in crs-setup.conf for instructions.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'XenForo exclusion rules', 'rule_file': 'REQUEST-903.9006-XENFORO-EXCLUSION-RULES.conf', + 'desc': 'These exclusions remedy false positives in a default XenForo install. The exclusions are only active ' + 'if crs_exclusions_xenforo=1 is set. See rule 900130 in crs-setup.conf for instructions.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Common exceptions', 'rule_file': 'REQUEST-905-COMMON-EXCEPTIONS.conf', + 'desc': 'This file is used as an exception mechanism to remove common false positives that may be encountered.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'IP reputation', 'rule_file': 'REQUEST-910-IP-REPUTATION.conf', + 'desc': 'IP reputation rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Method enforcement', 'rule_file': 'REQUEST-911-METHOD-ENFORCEMENT.conf', + 'desc': 'Method enforcement rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'DDOS protection', 'rule_file': 'REQUEST-912-DOS-PROTECTION.conf', + 'desc': 'Anti-Automation rules to detect Denial of Service attacks.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Protocol enforcement', 'rule_file': 'REQUEST-920-PROTOCOL-ENFORCEMENT.conf', + 'desc': 'Some protocol violations are common in application layer attacks. Validating HTTP requests eliminates ' + 'a large number of application layer attacks. The purpose of this rules file is to enforce HTTP RFC ' + 'requirements that state how the client is supposed to interact with the server.', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Protocol attack', 'rule_file': 'REQUEST-921-PROTOCOL-ATTACK.conf', + 'desc': 'Protocol attack rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack LFI', 'rule_file': 'REQUEST-930-APPLICATION-ATTACK-LFI.conf', + 'desc': 'Application attack LFI rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack RCE', 'rule_file': 'REQUEST-932-APPLICATION-ATTACK-RCE.conf', + 'desc': 'Application attack RCE rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack PHP', 'rule_file': 'REQUEST-933-APPLICATION-ATTACK-PHP.conf', + 'desc': 'Application attack PHP rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack NodeJS', 'rule_file': 'REQUEST-934-APPLICATION-ATTACK-NODEJS.conf', + 'desc': 'Application attack NodeJS rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack SQLI', 'rule_file': 'REQUEST-942-APPLICATION-ATTACK-SQLI.conf', + 'desc': 'Application attack SQLI rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack session-fixation', 'rule_file': 'REQUEST-943-APPLICATION-ATTACK-SESSION-FIXATION.conf', + 'desc': 'Application attack session-fixation rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack JAVA', 'rule_file': 'REQUEST-944-APPLICATION-ATTACK-JAVA.conf', + 'desc': 'Application attack JAVA rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Application attack blocking evaluation', 'rule_file': 'REQUEST-949-BLOCKING-EVALUATION.conf', + 'desc': 'Application attack blocking evaluation rule.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Data leakages', 'rule_file': 'RESPONSE-950-DATA-LEAKAGES.conf', + 'desc': 'The paranoia level skip rules 950020, 950021 and 950022 have odd numbers not in sync with other paranoia ' + 'level skip rules in other. files. This is done to avoid rule id collisions with CRSv2. This is also true ' + 'for rule 950130.', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Data leakages SQL', 'rule_file': 'RESPONSE-951-DATA-LEAKAGES-SQL.conf', + 'desc': 'Data leakages SQL rule', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Data leakages JAVA', 'rule_file': 'RESPONSE-952-DATA-LEAKAGES-JAVA.conf', + 'desc': 'Data leakages JAVA rule', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Data leakages PHP', 'rule_file': 'RESPONSE-953-DATA-LEAKAGES-PHP.conf', + 'desc': 'Data leakages PHP rule', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Data leakages IIS', 'rule_file': 'RESPONSE-954-DATA-LEAKAGES-IIS.conf', + 'desc': 'Data leakages IIS rule', 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Blocking evaluation', 'rule_file': 'RESPONSE-959-BLOCKING-EVALUATION.conf', + 'desc': 'You should set the score to the proper threshold you would prefer. If kept at "@gt 0" it will work ' + 'similarly to previous Mod CRS rules and will create an event in the error_log file if there are any ' + 'rules that match. If you would like to lessen the number of events generated in the error_log file, ' + 'you should increase the anomaly score threshold to something like "@gt 20". This would only generate ' + 'an event in the error_log file if there are multiple lower severity rule matches or if any 1 higher ' + 'severity item matches. You should also set the desired disruptive action (deny, redirect, etc...).', + 'service': 'nginx'}, + {'serv': serv, 'rule_name': 'Correlation', 'rule_file': 'RESPONSE-980-CORRELATION.conf', + 'desc': 'This file is used in post processing after the response has been sent to the client (in the logging phase). ' + 'Its purpose is to provide inbound+outbound correlation of events to provide a more intelligent designation ' + 'as to the outcome or result of the transaction - meaning, was this a successful attack?', + 'service': 'nginx'}, + ] + try: + WafRules.insert_many(data_source).execute() + except Exception as e: + out_error(e) + else: + return True + + +def select_waf_rules(serv, service): + query = WafRules.select(WafRules.id, WafRules.rule_name, WafRules.en, WafRules.desc).where( + (WafRules.serv == serv) + & (WafRules.service == service) + ) try: query_res = query.execute() except Exception as e: @@ -1425,6 +1541,22 @@ def update_enable_waf_rules(rule_id, serv, en): out_error(e) +def insert_new_waf_rule(rule_name: str, rule_file: str, rule_description: str, service: str, serv: str) -> int: + try: + last_id = WafRules.insert( + serv=serv, + rule_name=rule_name, + rule_file=rule_file, + desc=rule_description, + service=service + ).execute() + except Exception as e: + out_error(e) + else: + return last_id + + + def delete_waf_server(server_id): query = Waf.delete().where(Waf.server_id == server_id) try: diff --git a/app/templates/ajax/overivewWaf.html b/app/templates/ajax/overivewWaf.html index c40c442f..8a8c2b80 100644 --- a/app/templates/ajax/overivewWaf.html +++ b/app/templates/ajax/overivewWaf.html @@ -2,6 +2,11 @@ {% do waf_modes.append("On") %} {% do waf_modes.append("Off") %} {% do waf_modes.append("DetectionOnly") %} +{% if waf_service == 'haproxy' %} +{% set service_name = 'waf' %} +{% else %} +{% set service_name = 'waf_nginx' %} +{% endif %} {% for service in service_status %} {% if service.5|int() >= 1 %} @@ -17,18 +22,25 @@ {{ service.0 }} {% endif %} +{{service.3}} {% if service.3 == "On" or service.3 == "Off" or service.3 == "DetectionOnly" %} {% if role <= 2 %} - + - + + {% if waf_service == 'haproxy' %} + {% else %} + + + + {% endif %} {% endif %} @@ -46,6 +58,7 @@ {{ service.3 }} {% endif %} + {% if waf_service == 'haproxy' %} {% if service.4|int() == 1 %} @@ -53,13 +66,18 @@ {% endif %} + {% endif %} {% if role <= 2 %} - Open + Open {% endif %} + {% if waf_service == 'haproxy' %} View + {% elif waf_service == 'nginx' %} + View + {% endif %} {% else %} diff --git a/app/templates/base.html b/app/templates/base.html index e27eb582..c3f55fd7 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -93,6 +93,7 @@ {% if role <= 3 %}
  • Versions
  • SSL
    • +
  • WAF
    • {% endif %} @@ -276,7 +277,7 @@ Help SD Contacts - Cabinet + Cabinet Legal diff --git a/app/templates/waf.html b/app/templates/waf.html index c54912e5..c0578582 100644 --- a/app/templates/waf.html +++ b/app/templates/waf.html @@ -25,14 +25,29 @@ {{r.desc}} - View + View/Edit {% endfor %} +

    + Add



    + {% elif waf_rule_file %}
    +

    Config {{waf_rule_file}} from {{ serv }}

    @@ -41,13 +56,22 @@ - + +

    Back + {% if role <= 3 %} + + {% if service == 'haproxy' %} + + {% elif service == 'nginx' %} + + {% endif %} + {% endif %}

    @@ -59,7 +83,7 @@ {% else %} @@ -93,7 +117,9 @@ Server Actions WAF mode + {% if service == 'haproxy' %} Metrics + {% endif %} Manage rules Log @@ -106,11 +132,12 @@ {% endfor %} - + {% if service == 'haproxy' %} +
    Time range: