From f272ec4ca2e27c837c33e262305e3c31fcd97e27 Mon Sep 17 00:00:00 2001 From: Aidaho12 Date: Fri, 17 Aug 2018 10:41:50 +0600 Subject: [PATCH] v3.1 Meet Web application firewall! --- README.md | 4 +- app/add.py | 10 +- app/funct.py | 41 ++++++- app/options.py | 7 +- app/ovw.py | 3 +- app/scripts/install_haproxy.sh | 6 +- app/scripts/syn_flood_protect.sh | 15 +-- app/scripts/waf.sh | 188 +++++++++++++++++++++++++++++++ app/templates/add.html | 3 + app/templates/base.html | 2 +- app/templates/ihap.html | 4 + app/templates/ovw.html | 5 +- inc/script.js | 29 ++++- inc/users.js | 12 +- 14 files changed, 296 insertions(+), 33 deletions(-) create mode 100644 app/scripts/waf.sh diff --git a/README.md b/README.md index 16b4247..e3bb98e 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Support the project 21. Alerting about HAProxy service state 22. Metrics incoming connections 23. Web acceleration settings -24. Some functions WAF +24. Web application firewall ![alt text](image/haproxy-wi-metrics.jpeg "Merics") @@ -107,7 +107,7 @@ Login http://haproxy-wi-server/users.py, and add: users, groups and servers. Def ![alt text](image/haproxy-wi-admin-area.jpeg "Admin area") -For Runtime API enable state file on HAproxt servers and need install socat on all haproxy servers, and configre HAProxy: +For Runtime API, Metrics and Alerting enable state file and stat socket on HAproxt servers and need install socat on all haproxy servers, and configre HAProxy: ``` global stats socket *:1999 level admin diff --git a/app/add.py b/app/add.py index d8fb83d..b55b66c 100644 --- a/app/add.py +++ b/app/add.py @@ -36,6 +36,7 @@ print(output_from_parsed_template) hap_configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir') cert_path = sql.get_setting('cert_path') +haproxy_dir = sql.get_setting('haproxy_dir') if form.getvalue('mode') is not None: serv = form.getvalue('serv') @@ -111,7 +112,7 @@ if form.getvalue('mode') is not None: options_split += " option http-pretend-keepalive\n" if form.getvalue('blacklist') is not None: - options_split += " tcp-request connection reject if { src -f /etc/haproxy/black/"+form.getvalue('blacklist')+" }\n" + options_split += " tcp-request connection reject if { src -f "+haproxy_dir+"/black/"+form.getvalue('blacklist')+" }\n" if form.getvalue('cookie'): cookie = " cookie "+form.getvalue('cookie_name') @@ -165,8 +166,13 @@ if form.getvalue('mode') is not None: if cache == "2": cache_s = " http-request cache-use "+end_name+"\n http-response cache-store "+end_name+"\n" cache_set = "cache "+end_name+"\n total-max-size 4\n max-age 240\n" + + waf = "" + if form.getvalue('waf') is not None: + waf = " filter spoe engine modsecurity config "+haproxy_dir+"/spoe-modsecurity.conf\n" + waf += " http-request deny if { var(txn.modsec.code) -m int gt 0 }\n" - config_add = name + "\n" + bind + mode + "\n" + balance + options_split + filter + compression_s + cache_s + backend + servers_split + "\n" + cache_set + config_add = name + "\n" + bind + mode + "\n" + balance + options_split + filter + compression_s + cache_s + waf + backend + servers_split + "\n" + cache_set cfg = hap_configs_dir + serv + "-" + funct.get_data('config') + ".cfg" funct.get_config(serv, cfg) diff --git a/app/funct.py b/app/funct.py index 99a9364..b8ef1b3 100644 --- a/app/funct.py +++ b/app/funct.py @@ -275,6 +275,9 @@ def install_haproxy(serv, **kwargs): if kwargs.get('syn_flood') == "1": syn_flood_protect(serv) + if kwargs.get('waf') == "1": + waf_install(serv) + os.system("rm -f %s" % script) def syn_flood_protect(serv, **kwargs): @@ -296,6 +299,34 @@ def syn_flood_protect(serv, **kwargs): os.system("rm -f %s" % script) +def waf_install(serv, **kwargs): + import sql + script = "waf.sh" + tmp_config_path = sql.get_setting('tmp_config_path') + proxy = sql.get_setting('proxy') + haproxy_dir = sql.get_setting('haproxy_dir') + ver = check_haproxy_version(serv) + + os.system("cp scripts/%s ." % script) + + commands = [ "chmod +x "+tmp_config_path+script+" && " +tmp_config_path+script +" PROXY=" + proxy+ + " HAPROXY_PATH="+haproxy_dir +" VERSION="+ver ] + + upload(serv, tmp_config_path, script) + ssh_command(serv, commands, print_out="1") + + os.system("rm -f %s" % script) + +def check_haproxy_version(serv): + import sql + haproxy_sock_port = sql.get_setting('haproxy_sock_port') + ver = "" + cmd="echo 'show info' |nc %s %s |grep Version |awk '{print $2}'" % (serv, haproxy_sock_port) + output, stderr = subprocess_execute(cmd) + for line in output: + ver = line + return ver + def upload(serv, path, file, **kwargs): full_path = path + file @@ -399,7 +430,7 @@ def server_status(stdout): proc_count = "" for line in stdout: - if "Ncat: Connection refused." not in line: + if "Ncat: " not in line: for k in line: proc_count = k.split(":")[1] else: @@ -417,8 +448,6 @@ def ssh_command(serv, commands, **kwargs): if kwargs.get("ip") == "1": show_ip(stdout) - elif kwargs.get("compare") == "1": - compare(stdout) elif kwargs.get("show_log") == "1": show_log(stdout) elif kwargs.get("server_status") == "1": @@ -448,7 +477,9 @@ def subprocess_execute(cmd): def show_backends(serv, **kwargs): import json - cmd='echo "show backend" |nc %s 1999' % serv + import sql + haproxy_sock_port = sql.get_setting('haproxy_sock_port') + cmd='echo "show backend" |nc %s %s' % (serv, haproxy_sock_port) output, stderr = subprocess_execute(cmd) ret = "" for line in output: @@ -491,6 +522,4 @@ def get_files(**kwargs): return sorted(return_files, reverse=True) else: return files - - \ No newline at end of file diff --git a/app/options.py b/app/options.py index ee38078..1aed18d 100644 --- a/app/options.py +++ b/app/options.py @@ -254,7 +254,7 @@ if form.getvalue('servaction') is not None: command = [ cmd ] if enable != "show": - print('

You %s %s on HAproxy %s. Look it or Edit something else


' % (enable, backend, serv, serv)) + print('

You %s %s on HAproxy %s. Look it or Edit something else


' % (enable, backend, serv, serv)) funct.ssh_command(serv, command, show_log="1") action = 'edit.py ' + enable + ' ' + backend @@ -374,7 +374,7 @@ if form.getvalue('masteradd'): os.system("rm -f %s" % script) if form.getvalue('haproxyaddserv'): - funct.install_haproxy(form.getvalue('haproxyaddserv'), syn_flood=form.getvalue('syn_flood')) + funct.install_haproxy(form.getvalue('haproxyaddserv'), syn_flood=form.getvalue('syn_flood'), waf=form.getvalue('waf')) if form.getvalue('table_metrics'): import http.cookies @@ -475,8 +475,7 @@ if form.getvalue('metrics'): show(grid) if form.getvalue('get_hap_v'): - commands = [ "haproxy -v |grep ver|awk '{print $3}'" ] - output = funct.ssh_command(serv, commands) + output = funct.check_haproxy_version(serv) print(output) if form.getvalue('bwlists'): diff --git a/app/ovw.py b/app/ovw.py index 3acf823..fa740c1 100644 --- a/app/ovw.py +++ b/app/ovw.py @@ -17,12 +17,13 @@ def get_overview(): listhap = sql.get_dick_permit() commands = [ "ls -l %s |awk '{ print $6\" \"$7\" \"$8}'" % haproxy_config_path ] + commands1 = [ "ps ax |grep waf/bin/modsecurity |grep -v grep |wc -l" ] servers = [] for server in listhap: server_status = () cmd = 'echo "show info" |nc %s %s |grep -e "Process_num"' % (server[2], haproxy_sock_port) - server_status = (server[1],server[2], funct.server_status(funct.subprocess_execute(cmd)), funct.ssh_command(server[2], commands)) + server_status = (server[1],server[2], funct.server_status(funct.subprocess_execute(cmd)), funct.ssh_command(server[2], commands), funct.ssh_command(server[2], commands1)) servers.append(server_status) template = template.render(service_status = servers, role = sql.get_user_role_by_uuid(user_id.value)) diff --git a/app/scripts/install_haproxy.sh b/app/scripts/install_haproxy.sh index 33bddbd..2074f09 100644 --- a/app/scripts/install_haproxy.sh +++ b/app/scripts/install_haproxy.sh @@ -35,14 +35,14 @@ fi if hash apt-get 2>/dev/null; then sudo apt-get install haproxy socat -y else - sudo wget http://cbs.centos.org/kojifiles/packages/haproxy/1.8.1/5.el7/x86_64/haproxy18-1.8.1-5.el7.x86_64.rpm + wget http://cbs.centos.org/kojifiles/packages/haproxy/1.8.1/5.el7/x86_64/haproxy18-1.8.1-5.el7.x86_64.rpm sudo yum install haproxy18-1.8.1-5.el7.x86_64.rpm -y fi if [ $? -eq 1 ] then sudo yum install wget socat -y > /dev/null - sudo wget http://cbs.centos.org/kojifiles/packages/haproxy/1.8.1/5.el7/x86_64/haproxy18-1.8.1-5.el7.x86_64.rpm + wget http://cbs.centos.org/kojifiles/packages/haproxy/1.8.1/5.el7/x86_64/haproxy18-1.8.1-5.el7.x86_64.rpm sudo yum install haproxy18-1.8.1-5.el7.x86_64.rpm -y fi if [ $? -eq 1 ] @@ -103,7 +103,7 @@ sudo sed -i 's/#$UDPServerRun 514/$UDPServerRun 514/g' /etc/rsyslog.conf sudo sed -i 's/#$ModLoad imudp/$ModLoad imudp/g' /etc/rsyslog.conf sudo firewall-cmd --zone=public --add-port=8085/tcp --permanent -sudo sudo firewall-cmd --reload +sudo firewall-cmd --reload sudo setenforce 0 sudo sed -i 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config sudo systemctl enable haproxy diff --git a/app/scripts/syn_flood_protect.sh b/app/scripts/syn_flood_protect.sh index b232dba..cd6a78c 100644 --- a/app/scripts/syn_flood_protect.sh +++ b/app/scripts/syn_flood_protect.sh @@ -1,8 +1,9 @@ #!/bin/bash if [[ $1 == "enable" ]]; then - if grep -q "net.ipv4.tcp_syncookie = 1" /etc/sysctl.conf; then + if sudo grep -q "net.ipv4.tcp_syncookies = 1" /etc/sysctl.conf; then echo "SYN flood protectd allready enabled" + exit 1 else sudo bash -c cat <> /etc/sysctl.conf # Protection SYN flood @@ -11,16 +12,16 @@ net.ipv4.conf.all.rp_filter = 1 net.ipv4.tcp_max_syn_backlog = 1024 EOF - sudo sysctl -w net.ipv4.tcp_syncookies=1 - sudo sysctl -w net.ipv4.conf.all.rp_filter=1 - sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 - sudo sysctl -w net.ipv4.tcp_synack_retries=3 + sudo sysctl -w net.ipv4.tcp_syncookies=1 + sudo sysctl -w net.ipv4.conf.all.rp_filter=1 + sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 + sudo sysctl -w net.ipv4.tcp_synack_retries=3 fi fi if [[ $1 == "disable" ]]; then - sed -i 's/net.ipv4.tcp_max_syn_backlog = 1024/net.ipv4.tcp_max_syn_backlog = 256/' /etc/sysctl.conf - sed -i 's/net.ipv4.tcp_synack_retries = 3/net.ipv4.tcp_synack_retries = 5/' /etc/sysctl.conf + sudo sed -i 's/net.ipv4.tcp_max_syn_backlog = 1024/net.ipv4.tcp_max_syn_backlog = 256/' /etc/sysctl.conf + sudo sed -i 's/net.ipv4.tcp_synack_retries = 3/net.ipv4.tcp_synack_retries = 5/' /etc/sysctl.conf sudo sysctl -w net.ipv4.tcp_max_syn_backlog=256 sudo sysctl -w net.ipv4.tcp_synack_retries=5 fi \ No newline at end of file diff --git a/app/scripts/waf.sh b/app/scripts/waf.sh new file mode 100644 index 0000000..f28dcdc --- /dev/null +++ b/app/scripts/waf.sh @@ -0,0 +1,188 @@ +#!/bin/bash + +for ARGUMENT in "$@" +do + + KEY=$(echo $ARGUMENT | cut -f1 -d=) + VALUE=$(echo $ARGUMENT | cut -f2 -d=) + + case "$KEY" in + PROXY) PROXY=${VALUE} ;; + VERSION) VERSION=${VALUE} ;; + HAPROXY_PATH) HAPROXY_PATH=${VALUE} ;; + *) + esac + + +done + +VERSION_MAJ=$(echo $VERSION | awk -F"." '{print $1"."$2}') + +if [[ $PROXY != "" ]] +then + export http_proxy="$PROXY" + export https_proxy="$PROXY" + echo "Exporting proxy" +fi + +if [ -f $HAPROXY_PATH/waf/modsecurity.conf ];then + echo -e 'error: Haproxy WAF already installed. You can edit confighere

' + exit 1 +fi +wget -O /tmp/yajl-devel-2.0.4-4.el7.x86_64.rpm http://rpmfind.net/linux/centos/7.5.1804/os/x86_64/Packages/yajl-devel-2.0.4-4.el7.x86_64.rpm +wget -O /tmp/libevent-devel-2.0.21-4.el7.x86_64.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/libevent-devel-2.0.21-4.el7.x86_64.rpm +wget -O /tmp/modsecurity-2.9.2.tar.gz https://www.modsecurity.org/tarball/2.9.2/modsecurity-2.9.2.tar.gz +sudo yum install /tmp/libevent-devel-2.0.21-4.el7.x86_64.rpm /tmp/yajl-devel-2.0.4-4.el7.x86_64.rpm httpd-devel libxml2-devel gcc curl-devel -y + +if [ $? -eq 1 ]; then + echo -e "Can't download waf application. Check Internet connection" + exit 1 +fi +cd /tmp +sudo tar xf modsecurity-2.9.2.tar.gz +cd /tmp/modsecurity-2.9.2 +sudo ./configure --prefix=/tmp/modsecurity-2.9.2 --enable-standalone-module --disable-mlogc --enable-pcre-study --without-lua --enable-pcre-jit +sudo make +sudo make -C standalone install +if [ $? -eq 1 ]; then + echo -e "Can't compile waf application" + exit 1 +fi +sudo mkdir -p /tmp/modsecurity-2.9.2/INSTALL/include +sudo cp standalone/.libs/* /tmp/modsecurity-2.9.2/INSTALL/include +sudo cp standalone/* /tmp/modsecurity-2.9.2/INSTALL/include +sudo cp apache2/*.h /tmp/modsecurity-2.9.2/INSTALL/include + +wget -O /tmp/haproxy-$VERSION.tar.gz http://www.haproxy.org/download/$VERSION_MAJ/src/haproxy-$VERSION.tar.gz + +if [ $? -eq 1 ]; then + echo -e "Can't download Haproxy application. Check Internet connection" + exit 1 +fi +cd /tmp +sudo tar xf /tmp/haproxy-$VERSION.tar.gz +sudo mkdir $HAPROXY_PATH/waf +sudo mkdir $HAPROXY_PATH/waf/bin +sudo mkdir $HAPROXY_PATH/waf/rules +cd /tmp/haproxy-$VERSION/contrib/modsecurity +sudo make MODSEC_INC=/tmp/modsecurity-2.9.2/INSTALL/include MODSEC_LIB=/tmp/modsecurity-2.9.2/INSTALL/include APACHE2_INC=/usr/include/httpd/ APR_INC=/usr/include/apr-1 +if [ $? -eq 1 ]; then + echo -e "Can't compile waf application" + exit 1 +fi +sudo mv /tmp/haproxy-$VERSION/contrib/modsecurity/modsecurity $HAPROXY_PATH/waf/bin +wget -O $HAPROXY_PATH/waf/modsecurity.conf https://github.com/SpiderLabs/ModSecurity/raw/v2/master/modsecurity.conf-recommended + +sudo bash -c cat << EOF >> $HAPROXY_PATH/waf/modsecurity.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_10_ignore_static.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_10_setup.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_11_avs_traffic.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_11_brute_force.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_11_dos_protection.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_13_xml_enabler.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_16_authentication_tracking.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_16_scanner_integration.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_16_username_tracking.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_16_username_tracking.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_20_protocol_violations.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_21_protocol_anomalies.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_23_request_limits.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_25_cc_known.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_25_cc_track_pan.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_30_http_policy.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_35_bad_robots.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_40_generic_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_40_http_parameter_pollution.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_41_sql_injection_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_41_xss_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_42_comment_spam.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_42_tight_security.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_45_trojans.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_46_av_scanning.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_46_scanner_integration.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_46_slr_et_xss_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_46_slr_et_lfi_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_46_slr_et_sqli_attacks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_47_common_exceptions.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_49_inbound_blocking.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_50_outbound.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_55_marketing.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_56_pvi_checks.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_59_outbound_blocking.conf +Include $HAPROXY_PATH/waf/rules/modsecurity_crs_60_correlation.conf +EOF + +wget -O $HAPROXY_PATH/waf/unicode.mapping https://github.com/SpiderLabs/ModSecurity/raw/v2/master/unicode.mapping +wget -O /tmp/owasp.tar.gz https://github.com/SpiderLabs/owasp-modsecurity-crs/archive/2.2.9.tar.gz +cd /tmp/ +sudo tar xf /tmp/owasp.tar.gz +sudo mv /tmp/owasp-modsecurity-crs-2.2.9/modsecurity_crs_10_setup.conf.example $HAPROXY_PATH/waf/rules/modsecurity_crs_10_setup.conf +sudo mv /tmp/owasp-modsecurity-crs-2.2.9/*rules/* $HAPROXY_PATH/waf/rules/ +sudo sed -i 's/#SecAction/SecAction/' $HAPROXY_PATH/waf/rules/modsecurity_crs_10_setup.conf +sudo rm -f /tmp/owasp.tar.gz + +sudo bash -c cat << EOF > /etc/systemd/system/multi-user.target.wants/waf.service +[Unit] +Description=Defender WAF +After=syslog.target network.target + +[Service] +ExecStart=$HAPROXY_PATH/waf/bin/modsecurity -n 4 -f $HAPROXY_PATH/waf/modsecurity.conf +ExecReload=/bin/kill -USR2 $MAINPID +KillMode=mixed + +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=waf + +[Install] +WantedBy=multi-user.target +EOF + +sudo bash -c cat << EOF > /etc/rsyslog.d/waf.conf +if $programname startswith 'waf' then /var/log/waf.log +& stop +EOF + +sudo bash -c cat << EOF > $HAPROXY_PATH/spoe-modsecurity.conf +[modsecurity] +spoe-agent modsecurity-agent + messages check-request + option var-prefix modsec + timeout hello 100ms + timeout idle 30s + timeout processing 15ms + use-backend spoe-modsecurity + +spoe-message check-request + args unique-id method path query req.ver req.hdrs_bin req.body_size req.body + event on-frontend-http-request +EOF +if sudo grep -q "backend spoe-modsecurity" $HAPROXY_PATH/haproxy.cfg; then + echo -e "Backend for WAF exists" +else + sudo bash -c cat << EOF >> $HAPROXY_PATH/haproxy.cfg + +backend spoe-modsecurity + mode tcp + timeout connect 5s + timeout server 3m + server waf 127.0.0.1:12345 check +EOF +fi + +sudo systemctl daemon-reload +sudo systemctl enable waf +sudo systemctl restart waf +sudo rm -f /tmp/libevent-devel-2.0.21-4.el7.x86_64.rpm +sudo rm -f /tmp/modsecurity-2.9.2.tar.gz +sudo rm -f /tmp/yajl-devel-2.0.4-4.el7.x86_64.rpm +sudo rm -rf /tmp/haproxy-$VERSION +sudo rm -rf /tmp/haproxy-$VERSION.tar.gz +sudo rm -rf /tmp/modsecurity-2.9.2 + +if [ $? -eq 1 ]; then + echo "error: Can't start Haproxy WAF service

" + exit 1 +fi +echo "success" \ No newline at end of file diff --git a/app/templates/add.html b/app/templates/add.html index 57706cb..e788c0a 100644 --- a/app/templates/add.html +++ b/app/templates/add.html @@ -89,6 +89,7 @@ + + {% if add %}

{{ add }} was success added

@@ -529,6 +531,7 @@ 

 	

 	
+