From 8ac6081555dd95574692e93c0c985a28582c6595 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 31 Oct 2013 01:23:00 +1100 Subject: [PATCH 01/36] ENH: fix to use upstream --remove-rules https://fedorahosted.org/firewalld/ticket/10 --- config/action.d/firewall-cmd-direct-new.conf | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/config/action.d/firewall-cmd-direct-new.conf b/config/action.d/firewall-cmd-direct-new.conf index ac06aa57..0f7388a2 100644 --- a/config/action.d/firewall-cmd-direct-new.conf +++ b/config/action.d/firewall-cmd-direct-new.conf @@ -15,15 +15,8 @@ actionstart = firewall-cmd --direct --add-chain ipv4 filter fail2ban- firewall-cmd --direct --add-rule ipv4 filter fail2ban- 1000 -j RETURN firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- -# The following rule does not work, because firewalld keeps its own database of firewall rules. -# firewall-cmd --direct --passthrough ipv4 -F fail2ban- -# The better rule would be the following, but firewall-cmd has not implemented this command with firewalld-0.3.3-2.fc19 . -# firewall-cmd --direct --flush-chain ipv4 filter fail2ban- -# The following is a workaround using a loop to implement the --flush-chain command. -# https://fedorahosted.org/firewalld/ticket/10 - actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- - ( IFS='|' ; for r in $( firewall-cmd --direct --get-rules ipv4 filter fail2ban- | tr '\n' '|' ) ; do eval firewall-cmd --direct --remove-rule ipv4 filter fail2ban- $r ; done ) + firewall-cmd --direct --remove-rules ipv4 filter fail2ban- firewall-cmd --direct --remove-chain ipv4 filter fail2ban- actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'fail2ban-[ \t]' From 5eddd5d12dbb7a4d82d74598df2c9efb548f2e12 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 31 Oct 2013 09:10:59 +1100 Subject: [PATCH 02/36] DOC: document required firewalld version as > 0.3.7.1 --- config/action.d/firewall-cmd-direct-new.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/firewall-cmd-direct-new.conf b/config/action.d/firewall-cmd-direct-new.conf index 0f7388a2..3fd4e52b 100644 --- a/config/action.d/firewall-cmd-direct-new.conf +++ b/config/action.d/firewall-cmd-direct-new.conf @@ -3,7 +3,8 @@ # Author: Edgar Hoch # Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. # It uses "firewall-cmd" instead of "iptables". -# firewall-cmd is based on the command of version firewalld-0.3.4-1.fc19. +# +# Because of the --remove-rules in stop it requires a version AFTER (but not including) 0.3.7.1 [INCLUDES] From ee1edfbf0c7ad641836f93ce46f252c92cd6515c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 4 Nov 2013 17:51:41 +1100 Subject: [PATCH 03/36] BF: remove duplication definition secion in webmin-auth --- config/filter.d/webmin-auth.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/filter.d/webmin-auth.conf b/config/filter.d/webmin-auth.conf index 18bf6361..a0f014cd 100644 --- a/config/filter.d/webmin-auth.conf +++ b/config/filter.d/webmin-auth.conf @@ -9,8 +9,6 @@ before = common.conf _daemon = webmin -[Definition] - failregex = ^%(__prefix_line)sNon-existent login as .+ from \s*$ ^%(__prefix_line)sInvalid login as .+ from \s*$ From 87f68d7564ff375eda6c423b28ca09e3f53123ae Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 6 Nov 2013 11:37:56 +1100 Subject: [PATCH 04/36] firewalld-0.3.8 release that support --remove-rules out so documenting this. --- ChangeLog | 1 + config/action.d/firewall-cmd-direct-new.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f77d9491..623d998e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -54,6 +54,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Edgar Hoch * action.d/firewall-cmd-direct-new.conf - action for firewalld from https://bugzilla.redhat.com/show_bug.cgi?id=979622 + NOTE: requires firewalld-0.3.8+ Andy Fragen and Daniel Black * filter.d/osx-ipfw.conf - ipfw action for OSX based on random rule numbers. diff --git a/config/action.d/firewall-cmd-direct-new.conf b/config/action.d/firewall-cmd-direct-new.conf index 3fd4e52b..55b6762d 100644 --- a/config/action.d/firewall-cmd-direct-new.conf +++ b/config/action.d/firewall-cmd-direct-new.conf @@ -4,7 +4,7 @@ # Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. # It uses "firewall-cmd" instead of "iptables". # -# Because of the --remove-rules in stop it requires a version AFTER (but not including) 0.3.7.1 +# Because of the --remove-rules in stop this action requires firewalld-0.3.8+ [INCLUDES] From d22214da798a9175bf0fec11fef1253534561baa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 6 Nov 2013 12:03:19 +1100 Subject: [PATCH 05/36] Add Fedora git repo of fail2ban package to DEVELOP --- DEVELOP | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOP b/DEVELOP index a8f8899f..bf0bb863 100644 --- a/DEVELOP +++ b/DEVELOP @@ -732,6 +732,7 @@ Which indicates that testcases/files/logs/mysqld.log has been moved or is a dire http://svnweb.freebsd.org/ports/head/security/py-fail2ban/Makefile?view=markup * Fedora: Axel Thimm https://apps.fedoraproject.org/packages/fail2ban + http://pkgs.fedoraproject.org/cgit/fail2ban.git * Gentoo: netmon@gentoo.org http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/metadata.xml?view=markup * openSUSE: Stephan Kulow From 8b54523316022d2f4e017f353ad1892f06b6bd8e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 6 Nov 2013 12:13:37 +1100 Subject: [PATCH 06/36] BF: fix to filter.d/wuftp to support pam authentication - Debian bug #665925 --- ChangeLog | 2 ++ config/filter.d/wuftpd.conf | 3 +++ testcases/files/logs/wuftpd | 2 ++ 3 files changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index 6497a731..4fcc7cd0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -80,6 +80,8 @@ IMPORTANT incompatible changes: * filter.d/mysqld-auth.conf - mysql can use syslog * filter.d/sshd - regex enhancements to support openssh-6.3. Closes Debian bug #722970 + * filter.d/wuftpd - regex enhancements to support pam and wuftpd. Closes + Debian bug #665925 Rolf Fokkens * action.d/dshield.conf and complain.conf -- reorder mailx arguments. https://bugzilla.redhat.com/show_bug.cgi?id=998020 diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 942de82a..45149f60 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -11,8 +11,11 @@ before = common.conf [Definition] _daemon = wu-ftpd +__pam_re=\(?pam_unix(?:\(wu-ftpd:auth\))?\)?:? failregex = ^%(__prefix_line)sfailed login from \S+ \[\]\s*$ + ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=(?:\s+user=.*)?\s*$ + ignoreregex = diff --git a/testcases/files/logs/wuftpd b/testcases/files/logs/wuftpd index bbb816cc..948e848f 100644 --- a/testcases/files/logs/wuftpd +++ b/testcases/files/logs/wuftpd @@ -3,3 +3,5 @@ Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] # failJSON: { "time": "2004-10-11T16:45:07", "match": true , "host": "198.51.100.71" } Oct 11 16:45:07 ubuntu wu-ftpd[2360]: failed login from example.com [198.51.100.71] +# failJSON: { "time": "2005-03-22T09:35:02", "match": true , "host": "198.51.100.71" } +Mar 22 09:35:02 SiD wu-ftpd[31278]: pam_unix(wu-ftpd:auth): authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=198.51.100.71 user=root From e55b24c533d105cb284c414a2df34b7b48eb3680 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 6 Nov 2013 12:51:21 +1100 Subject: [PATCH 07/36] BF: fix dovecot filter for newer failure message. Closes Debian bug #709324 --- ChangeLog | 2 ++ config/filter.d/dovecot.conf | 2 +- testcases/files/logs/dovecot | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6497a731..9cacd141 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,8 @@ IMPORTANT incompatible changes: closes gh-266. hostsdeny supports daemon_list now too. * action.d/bsd-ipfw - action option unsed. Change blocktype to port unreach instead of deny for consistancy. + * filter.d/dovecot - added to support different dovecot failure + "..disallowed plaintext auth". Closes Debian bug #709324 * filter.d/roundcube-auth - timezone offset can be positive or negative * action.d/bsd-ipfw - action option unsed. Fixed to blocktype for consistency. default to port unreach instead of deny diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 2caa04b3..a51ce259 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use disabled \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ignoreregex = diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index d2aa59ca..aa79e65a 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -12,6 +12,9 @@ # failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 +# failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } +Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disallowed plaintext auth): rip=190.210.136.21, lip=113.212.99.193, session= + # failJSON: { "time": "2005-06-13T16:30:54", "match": true , "host": "49.176.98.87" } Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS # failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } From 5ebc38683355fbaf3137b8053b29fd74d513e22a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 6 Nov 2013 13:35:04 +1100 Subject: [PATCH 08/36] DOC: few more links for DEVELOP --- DEVELOP | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/DEVELOP b/DEVELOP index bf0bb863..9a3be94b 100644 --- a/DEVELOP +++ b/DEVELOP @@ -730,15 +730,20 @@ Which indicates that testcases/files/logs/mysqld.log has been moved or is a dire http://packages.qa.debian.org/f/fail2ban.html * FreeBSD: Christoph Theis theis@gmx.at>, Nick Hilliard http://svnweb.freebsd.org/ports/head/security/py-fail2ban/Makefile?view=markup + http://www.freebsd.org/cgi/query-pr-summary.cgi?text=fail2ban * Fedora: Axel Thimm https://apps.fedoraproject.org/packages/fail2ban http://pkgs.fedoraproject.org/cgit/fail2ban.git + https://admin.fedoraproject.org/pkgdb/acls/bugs/fail2ban * Gentoo: netmon@gentoo.org http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/metadata.xml?view=markup + https://bugs.gentoo.org/buglist.cgi?quicksearch=fail2ban * openSUSE: Stephan Kulow - https://build.opensuse.org/package/users?package=fail2ban&project=openSUSE%3AFactory + https://build.opensuse.org/package/show/openSUSE:Factory/fail2ban * Mac Ports: @Malbrouck on github (gh-49) https://trac.macports.org/browser/trunk/dports/security/fail2ban/Portfile + * Mageia: + https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban An potentially to the fail2ban-users directory. # Wait for feedback from distributors From f26fba9c19bdf319d8750f2c04e71b28edc29d79 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 6 Nov 2013 13:47:45 -0500 Subject: [PATCH 09/36] DOC: Untabifying and reindenting a bit ChangeLog --- ChangeLog | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4f69b4a1..87944c76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.11.pre1) 2013/10/30 +Fail2Ban (version 0.8.11.pre1) 2013/10/30 ================================================================================ ver. 0.8.11 (2013/11/XXX) - loves-unittests and tight, DoS free, filter regexes @@ -27,12 +27,12 @@ possibility that we have inadvertently, despite our best intentions, incorrectly allowed a failure to continue. We will fix this as quickly as humanly possible. -IMPORTANT incompatible changes: - Filter name changes: - * 'lighttpd-fastcgi' filter has been renamed to 'suhosin' - * 'sasl' has been renamed to 'postfix-sasl' - These will require changing in jail.{conf,local} if using these filters. - Exim filter has been split into an spam and a relay/auth filter. +- IMPORTANT incompatible changes: + Filter name changes: + * 'lighttpd-fastcgi' filter has been renamed to 'suhosin' + * 'sasl' has been renamed to 'postfix-sasl' + These will require changing in jail.{conf,local} if using these filters. + Exim filter has been split into an spam and a relay/auth filter. - Fixes: Daniel Black & Marcel Dopita @@ -110,7 +110,7 @@ IMPORTANT incompatible changes: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added * fail2ban-regex - now generates http://www.debuggex.com urls for debugging - regular expressions with the -D parameter. + regular expressions with the -D parameter. Daniel Black * filter.d/exim-spam.conf -- a splitout of exim's spam regexes with additions for greater control over filtering spam. @@ -131,8 +131,8 @@ IMPORTANT incompatible changes: * reorder parsing of jail.conf, jail.d/*.conf, jail.local, jail.d/*.local and likewise for fail2ban.{conf|local|d/*.conf|d/*.local}. Closes gh-392 * jail.conf now has asterisk jail - no need for asterisk-tcp and - asterisk-udp. Users should replace existing jails with asterisk to - reduce duplicate parsing of the asterisk log file. + asterisk-udp. Users should replace existing jails with asterisk to + reduce duplicate parsing of the asterisk log file. * filter.d/{suhosin,pam-generic,gssftpd,sogo-auth,webmin}- regex anchor at start * filter.d/vsftpd - anchored regex at start. disable old pam format regex From 28ee7ba12303faf1d6c81c60506d79a090099bae Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 6 Nov 2013 14:04:30 -0500 Subject: [PATCH 10/36] DOC: keeping Changelog release-phrases uniform, simplified intro, unified --- ChangeLog | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/ChangeLog b/ChangeLog index 87944c76..fc001187 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,36 +7,33 @@ Fail2Ban (version 0.8.11.pre1) 2013/10/30 ================================================================================ -ver. 0.8.11 (2013/11/XXX) - loves-unittests and tight, DoS free, filter regexes +ver. 0.8.11 (2013/11/XXX) - loves-unittests-and-tight-DoS-free-filter-regexes ----------- -In light of CVE-2013-2178 that triggered our last release we have put a -significant effort into tightening all of the regexs of our filters to avoid -another similar vulnerability. All filters have been updated and some to -include more failure regexs supporting previously unbanned failures and -support for newer application versions too. There are test cases for most log +In light of CVE-2013-2178 that triggered our last release we have put +a significant effort into tightening all of the regexs of our filters +to avoid another similar vulnerability. All filters have been updated +and some to catch more login/authentication failures and to support +for newer application versions. There are test cases for most log cases of failures now. -As usual if you have other examples that demonstrate that a filter is -insufficient please give us an example log line on the github issue tracker -http://github.com/fail2ban/fail2ban/issues and NOT on a random blog in some -obscure corner of the Internet. - -During the tightening of the regexs to avoid DoS vulnerabilities there is the -possibility that we have inadvertently, despite our best intentions, -incorrectly allowed a failure to continue. We will fix this as quickly as -humanly possible. +As usual, if you have other examples that demonstrate that a filter is +insufficient, or if we have inadvertently introduced a regression, +please provide us with example log lines on the github issue tracker +http://github.com/fail2ban/fail2ban/issues and NOT on a random blog in +some obscure corner of the Internet. - IMPORTANT incompatible changes: Filter name changes: * 'lighttpd-fastcgi' filter has been renamed to 'suhosin' * 'sasl' has been renamed to 'postfix-sasl' - These will require changing in jail.{conf,local} if using these filters. - Exim filter has been split into an spam and a relay/auth filter. + * 'exim' spam catching failregexes was split out into 'exim-spam' + These changes will require changing jail.{conf,local} if any of + those filters were used. - Fixes: Daniel Black & Marcel Dopita - * filter.d/apache-auth -- fixed and apache auth samples provide. closes #286 + * filter.d/apache-auth -- fixed and apache auth samples provide. Closes gh-286 Yaroslav Halchenko * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 * filter.d/apache-common.conf -- support apache 2.4 more detailed error @@ -62,8 +59,8 @@ humanly possible. * filter.d/asterisk -- more regexes Daniel Black * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across - all platforms to ensure permissions are the same before and after a ban - - closes gh-266. hostsdeny supports daemon_list now too. + all platforms to ensure permissions are the same before and after a ban. + Closes gh-266. hostsdeny supports daemon_list now too. * action.d/bsd-ipfw - action option unsed. Change blocktype to port unreach instead of deny for consistancy. * filter.d/dovecot - added to support different dovecot failure @@ -89,7 +86,7 @@ humanly possible. https://bugzilla.redhat.com/show_bug.cgi?id=998020 John Doe (ache) * action.d/bsd-ipfw.conf - invert actionstop logic to make exist status 0. - closes gh-343. + Closes gh-343. JP Espinosa (Reviewed by O.Poplawski) * files/redhat-initd - rewritten to use stock init.d functions thus avoiding problems with getpid. Also $network and iptables moved @@ -163,7 +160,7 @@ humanly possible. * filter.d/{courier{login,smtp},proftpd,sieve,wuftpd,xinetd} - General regex impovements Zurd - * filter.d/postfix - add filter for VRFY failures. closes gh-322. + * filter.d/postfix - add filter for VRFY failures. Closes gh-322. Orion Poplawski * fail2ban.d/ and jail.d/ directories are added to etc/fail2ban to facilitate their use From 6f321068f1c92a338aa08dfaa12db83a2ba475dd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 7 Nov 2013 14:25:57 -0800 Subject: [PATCH 11/36] NF: gen_badbots script to (re)generate/update config/filter.d/apache-badbots.conf --- files/gen_badbots | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 files/gen_badbots diff --git a/files/gen_badbots b/files/gen_badbots new file mode 100755 index 00000000..278058f7 --- /dev/null +++ b/files/gen_badbots @@ -0,0 +1,75 @@ +#!/bin/bash +#-------------------------- =+- Shell script -+= -------------------------- +# +# Yaroslav Halchenko CS@UNM, CS@NJIT +# web: http://www.onerussian.com & PSYCH@RUTGERS +# e-mail: yoh@onerussian.com ICQ#: 60653192 +# +# DESCRIPTION (NOTES): +# +# Script to fetch list of agent strings from http://www.user-agents.org +# which are known to be from mailicious bots, and create apache-badbots.conf +# filter for fail2ban +# +# COPYRIGHT: Yaroslav Halchenko 2007-2013 +# +# LICENSE: +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, +# MA 02110-1301, USA. +# +# On Debian system see /usr/share/common-licenses/GPL for the full license. +# +#-----------------\____________________________________/------------------ + +url=http://www.user-agents.org/index.shtml +badbots=$( +for f in "" "?g_m" "?moz" "?n_s" "?t_z"; do + wget -q -O- $url$f; +done \ +| grep -h -B4 'S '\ +| sed -e 's/ //g' \ +| awk '/^--/{getline; gsub(" ",""); print $0}' \ +| sed -e 's/\([.\:|()]\)/\\\1/g' \ +| uniq \ +| tr '\n' '|' \ +| sed -e 's/|$//g' +) + +echo $badbots >| /tmp/badbots.tmp + +cat >| config/filter.d/apache-badbots.conf < -.*"(GET|POST).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ + +ignoreregex = + +# DEV Notes: +# List of bad bots fetched from http://www.user-agents.org +# Generated on `date` by $0. +# +# Author: Yaroslav Halchenko +EOF From 452230835411fcc86624aa0096fbb74c69be33d6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 7 Nov 2013 14:26:18 -0800 Subject: [PATCH 12/36] ENH: regenerated config/filter.d/apache-badbots.conf --- config/filter.d/apache-badbots.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index 9ee44c69..b2ac9626 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -3,12 +3,12 @@ # Regexp to catch known spambots and software alike. Please verify # that it is your intent to block IPs which were driven by # above mentioned bots. - + [Definition] badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider -badbots = atSpider/1\.0|autoemailspider|China Local Browse 2\.6|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|MVAClient|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|sogou spider|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|WebVulnCrawl\.blogspot\.com/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 +badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, +http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 failregex = ^ -.*"(GET|POST).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ @@ -16,6 +16,6 @@ ignoreregex = # DEV Notes: # List of bad bots fetched from http://www.user-agents.org -# Generated on Sun Feb 11 01:09:15 EST 2007 by ./badbots.sh +# Generated on Thu Nov 7 14:23:35 PST 2013 by files/gen_badbots. # # Author: Yaroslav Halchenko From a148d35d705b84fd97f3c2e00d28483f2eb4b92c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 8 Nov 2013 10:06:40 +1100 Subject: [PATCH 13/36] ENH: add filter.d/nginx-http-auth. Partially forfills #405 --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/nginx-http-auth.conf | 15 +++++++++++++++ config/jail.conf | 7 +++++++ testcases/files/logs/nginx-http-auth | 6 ++++++ 5 files changed, 31 insertions(+) create mode 100644 config/filter.d/nginx-http-auth.conf create mode 100644 testcases/files/logs/nginx-http-auth diff --git a/ChangeLog b/ChangeLog index fc001187..813213ed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -112,6 +112,8 @@ some obscure corner of the Internet. * filter.d/exim-spam.conf -- a splitout of exim's spam regexes with additions for greater control over filtering spam. * add date expression for apache-2.4 - milliseconds + * filter.d/nginx-http-auth -- filter added for http basic authentication + failures in nginx. Partially forfills gh-405. Christophe Carles & Daniel Black * filter.d/perdition.conf -- filter added Mark McKinstry diff --git a/THANKS b/THANKS index e70ca9c9..5f6d1b2b 100644 --- a/THANKS +++ b/THANKS @@ -54,6 +54,7 @@ Michael Hanselmann Nick Munger Patrick Börjesson Raphaël Marichez +RealRancor René Berber Robert Edeker Rolf Fokkens diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf new file mode 100644 index 00000000..00f152b7 --- /dev/null +++ b/config/filter.d/nginx-http-auth.conf @@ -0,0 +1,15 @@ +# fail2ban filter configuration for nginx + + +[Definition] + + +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+" + +ignoreregex = + +# DEV NOTES: +# Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files +# Extensive search of all nginx auth failures not done yet. +# +# Author: Daniel Black diff --git a/config/jail.conf b/config/jail.conf index 23e30c83..4bda3c4a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -181,6 +181,13 @@ logpath = /var/log/apache*/*error.log maxretry = 6 +[nginx-http-auth] + +enabled = false +filter = nginx-http-auth +logpath = /var/log/nginx/error.log + + # The hosts.deny path can be defined with the "file" argument if it is # not in /etc. [postfix-tcpwrapper] diff --git a/testcases/files/logs/nginx-http-auth b/testcases/files/logs/nginx-http-auth new file mode 100644 index 00000000..0fa7a7bd --- /dev/null +++ b/testcases/files/logs/nginx-http-auth @@ -0,0 +1,6 @@ + +# failJSON: { "time": "2012-04-09T11:53:29", "match": true , "host": "192.0.43.10" } +2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" +# failJSON: { "time": "2012-04-09T11:53:36", "match": true , "host": "192.0.43.10" } +2012/04/09 11:53:36 [error] 2865#0: *66647 user "xyz": password mismatch, client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" + From ab9d921162038a22cbea48611c63b6e42f3a8b17 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 8 Nov 2013 10:09:19 +1100 Subject: [PATCH 14/36] BF: missed action in nginx-http-auth --- config/jail.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/jail.conf b/config/jail.conf index 4bda3c4a..486ea078 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -185,6 +185,7 @@ maxretry = 6 enabled = false filter = nginx-http-auth +action = iptables-multiport[name=nginx-http-auth,port="80,443"] logpath = /var/log/nginx/error.log From d7560d4041e90d99fd306ad5fb8d5d89424e45e0 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 8 Nov 2013 10:24:50 +1100 Subject: [PATCH 15/36] ENH: condense asterisk regexs for speed --- config/filter.d/asterisk.conf | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 35906d11..f77a1557 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -8,13 +8,7 @@ __pid_re = (?:\[\d+\]) # All Asterisk log messages begin like this: log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d* -failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ - ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ +failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|No matching peer found|Username/auth name mismatch|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ ^%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ ^%(log_prefix)s Host failed to authenticate as '[^']*'$ ^%(log_prefix)s No registration for peer '[^']*' \(from \)$ From eace931c19d7da6ae8a81de9db6258d605a749f2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 7 Nov 2013 15:47:25 -0800 Subject: [PATCH 16/36] Changelog for prior changes (gen_buildbots) --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index fc001187..3d9d1a96 100644 --- a/ChangeLog +++ b/ChangeLog @@ -149,6 +149,9 @@ some obscure corner of the Internet. * filter.d/roundcube-auth.conf -- anchored version * date matching - for standard asctime formats prefer more detailed first (thus use year if available) + * files/gen_badbots was added and filter.d/apache-badbots.conf was + regenerated to get updated (although now still an old) list of + "bad" bots Alexander Dietrich * action.d/sendmail-common.conf -- added common sendmail settings file and made the sender display name configurable From abb012ae5c90c4bfc2b83512bc168f1ecaa0d10e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 10:00:37 -0800 Subject: [PATCH 17/36] BF: fixing injection for OpenSSH 6.3 -- making .* before non-greedy --- config/filter.d/sshd.conf | 2 +- testcases/files/logs/sshd | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 08456177..be80b02f 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -14,7 +14,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .* from (?: port \d*)?(?: ssh\d*)?(: (ruser .{0,100}|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".{0,100}", client host ".{0,100}")?))?\s*$ + ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .{0,100}|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".{0,100}", client host ".{0,100}")?))?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 96338220..def0ebb5 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -94,3 +94,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: DSA 01:c0:79:41:91:31:9a:7d:95:23:91:ac:b1:6d:59:81, client user "dan", client host "localhost.localdomain" + +# Injecting into rhost for the format of OpenSSH >=6.3 +# failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } +Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 From 750e0c1e3dbce856437c115142d57f18b6c1fac7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 10:06:24 -0800 Subject: [PATCH 18/36] BF: disallow exploiting of non-greedy .* in previous fix by providing too long rhost -- do not impose length limits for user-provided input since daemon might eventually change reported length and we would need to adjust anyways. So limiting in length does not provide additional security but allows for a possible injection vector --- ChangeLog | 2 +- config/filter.d/sshd.conf | 2 +- testcases/files/logs/sshd | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3d9d1a96..5120c139 100644 --- a/ChangeLog +++ b/ChangeLog @@ -78,7 +78,7 @@ some obscure corner of the Internet. * filter.d/recidive -- support f2b syslog target and anchor regex at start * filter.d/mysqld-auth.conf - mysql can use syslog * filter.d/sshd - regex enhancements to support openssh-6.3. Closes Debian - bug #722970 + bug #722970. Thanks Colin Watson for the regex analysis. * filter.d/wuftpd - regex enhancements to support pam and wuftpd. Closes Debian bug #665925 Rolf Fokkens diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index be80b02f..d97fd675 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -14,7 +14,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .{0,100}|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".{0,100}", client host ".{0,100}")?))?\s*$ + ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index def0ebb5..4f862d89 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -95,6 +95,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: DSA 01:c0:79:41:91:31:9a:7d:95:23:91:ac:b1:6d:59:81, client user "dan", client host "localhost.localdomain" -# Injecting into rhost for the format of OpenSSH >=6.3 -# failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } +# failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting into rhost for the format of OpenSSH >=6.3" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 + +# failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } +Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 From bf245f9640e3a957f1deb751f9eb28742f957107 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 14:34:31 -0800 Subject: [PATCH 19/36] DOC: adding DEV Notes for for non-greedy matchin within sshd.conf --- config/filter.d/sshd.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index d97fd675..a36b050c 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -26,4 +26,11 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ignoreregex = +# DEV Notes: +# +# "Failed \S+ for .*? from ..." failregex uses non-greedy catch-all because +# it is coming before use of which is not hard-anchored at the end as well, +# and later catch-all's could contain user-provided input, which need to be greedily +# matched away first. +# # Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black From 49024fe6ea1283c1bec69a2547237dd1e4adf117 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 14:36:56 -0800 Subject: [PATCH 20/36] DOC: minor typos in ChangeLog --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 648f1e1a..27e598ef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -96,7 +96,7 @@ some obscure corner of the Internet. - New Features: Edgar Hoch - * action.d/firewall-cmd-direct-new.conf - action for firewalld + * action.d/firewall-cmd-direct-new.conf - action for firewalld from https://bugzilla.redhat.com/show_bug.cgi?id=979622 NOTE: requires firewalld-0.3.8+ Andy Fragen and Daniel Black @@ -113,7 +113,7 @@ some obscure corner of the Internet. with additions for greater control over filtering spam. * add date expression for apache-2.4 - milliseconds * filter.d/nginx-http-auth -- filter added for http basic authentication - failures in nginx. Partially forfills gh-405. + failures in nginx. Partially fulfills gh-405. Christophe Carles & Daniel Black * filter.d/perdition.conf -- filter added Mark McKinstry From ac061155f093464fb6cd2329d3d513b15c68e256 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 14:40:52 -0800 Subject: [PATCH 21/36] BF: anchor introduced nginx-http-auth at the end needed since request probably could be not a correct HTTP statement but continue with all those to match till the end and then injected ", client: VICTIM, server..." thus allowing injection. We better anchor at the end then --- config/filter.d/nginx-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 00f152b7..79dda30b 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+" +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"\s*$ ignoreregex = From 724c6bfd922638ecc1eddf9197df5369eeb7cd0b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 9 Nov 2013 10:35:13 +1100 Subject: [PATCH 22/36] DOC: filter regex debugging --- DEVELOP | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/DEVELOP b/DEVELOP index 9a3be94b..0e37a0c3 100644 --- a/DEVELOP +++ b/DEVELOP @@ -289,15 +289,19 @@ TIP: Some applications log spaces at the end. If you are not sure add \s*$ as the end part of the regex. If your regex is not matching, http://www.debuggex.com/?flavor=python can help -to tune it: +to tune it. fail2ban-regex -D ... will present Debuggex URLs for the regexs +and sample log files that you pass into it. +In general use when using regex debuggers for generating fail2ban filters: * use regex from the ./fail2ban-regex output (to ensure all substitutions are -done) and replace with (?&.ipv4). Make sure that regex type set to -Python; -* for the test data put your log output with the time removed; -- when you have fixed the regex put it back into your filter file. +done) +* replace with (?&.ipv4) +* make sure that regex type set to Python +* for the test data put your log output with the date/time removed -Please spread the good word about debuggex - Serge Toarca is kindly continuing +When you have fixed the regex put it back into your filter file. + +Please spread the good word about Debuggex - Serge Toarca is kindly continuing its free availability to Open Source developers. Finishing up: From b8f40fef1bf362f5645b912b651797e4d2b83de2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:08:10 +1100 Subject: [PATCH 23/36] DOC: more on filter regexes - DEVELOP --- DEVELOP | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/DEVELOP b/DEVELOP index 0e37a0c3..d776d8f7 100644 --- a/DEVELOP +++ b/DEVELOP @@ -331,7 +331,7 @@ failregex, while matching inserted text to the part, they have the ability to deny any host they choose. So the part must be anchored on text generated by the application, and -not the user, to a extent sufficient to prevent user inserting the entire text +not the user, to an extent sufficient to prevent user inserting the entire text matching this or any other failregex. Ideally filter regex should anchor at the beginning and at the end of log line. @@ -381,7 +381,7 @@ Note if we'd just had the expression: Then provided the user put a space in their command they would have never been banned. -2. Filter regex can match other user injected data +2. Unanchored regex can match other user injected data From the Apache vulnerability CVE-2013-2178 ( original ref: https://vndh.net/note:fail2ban-089-denial-service ). @@ -402,7 +402,82 @@ Now the log line will be: As this log line doesn't match other expressions hence it matches the above regex and blocks 192.168.33.1 as a denial of service from the HTTP requester. -3. Application generates two identical log messages with different meanings +3. Over greedy pattern matching + +From: https://github.com/fail2ban/fail2ban/pull/426 + +An example ssh log (simplified) + + Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser remoteuser + +As we assume username can include anything including spaces its prudent to put +.* here. The remote user can also exist as anything so lets not make assumptions again. + + failregex = ^%(__prefix_line)sFailed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + +So this works. The problem is if the .* after remote user is injected by the +user to be 'from 1.2.3.4'. The resultant log line is. + + Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 + +Testing with: + + fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$' + +TIP: I've removed the bit that matches __prefix_line from the regex and log. + +Shows: + + 1) [1] ^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + 1.2.3.4 Sun Sep 29 17:15:02 2013 + +It should of matched 127.0.0.1. So the first greedy part of the greedy regex +matched until the end of the string. The was no "from " so the regex +engine worked backwards from the end of the string until this was matched. + +The result was that 1.2.3.4 was matched, injected by the user, and the wrong IP +was banned. + +The solution here is to make the first .* non-greedy with .*?. Here it matches +as little as required and the fail2ban-regex tool shows the output: + + fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$' + + 1) [1] ^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + 127.0.0.1 Sun Sep 29 17:15:02 2013 + +So the general case here is a log line that contains: + + (fixed_data_1)(fixed_data_2)(user_injectable_data) + +Where the regex that matches fixed_data_1 is gready and matches the entire +string, before moving backwards and user_injectable_data can match the entire +string. + +Another case: + +ref: https://www.debuggex.com/r/CtAbeKMa2sDBEfA2/0 + +A webserver logs the following without URL escaping: + + [error] 2865#0: *66647 user "xyz" was not found in "/file", client: 1.2.3.1, server: www.host.com, request: "GET ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host", host: "www.myhost.com" + +regex: + + failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (?:password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ .+ HTTP/\d+\.\d+", host: "\S+" + +The .* matches to the end of the string. Finds that it can't continue to match +", client ... so it moves from the back and find that the user injected web URL: + + ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host + +In this case there is a fixed host: "www.myhost.com" at the end so the solution +is to anchor the regex at the end with a $. + +If this wasn't the case then first .* needed to be made so it didn't capture +beyond . + +4. Application generates two identical log messages with different meanings If the application generates the following two messages under different circumstances: From d955714d26ba78f11999b4bd99a7c36ff8c55679 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:11:32 +1100 Subject: [PATCH 24/36] TST: test case that shows injection --- testcases/files/logs/sshd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 4f862d89..ed4857bd 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -100,3 +100,6 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 + +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From 061a26c40815b8594e9ffce1e9056aeac7dff5cd Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:28:09 +1100 Subject: [PATCH 25/36] TST: fix space in sshd sample log --- testcases/files/logs/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index ed4857bd..541afb19 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -101,5 +101,5 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 -# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From d90130234dfba1fd389118941dcf4ec8db031c8b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:29:54 +1100 Subject: [PATCH 26/36] TST: end of json in sshd sample log --- testcases/files/logs/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 541afb19..3c50dcfd 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -101,5 +101,5 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 -# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From 87516eb92b5792ce1202a5d67da481f6019ad52a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 09:46:40 +1100 Subject: [PATCH 27/36] ENH: apache-overflows - more detail on "request failed: URI too long (longer than %d)" with test case --- config/filter.d/apache-overflows.conf | 8 +++++++- testcases/files/logs/apache-overflows | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index de1c770d..68669222 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,8 +8,14 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long|erroneous characters after protocol string) +failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long \(longer than \d+\)|erroneous characters after protocol string) ignoreregex = +# DEV Noptes: +# +# fgrep -r 'URI too long' httpd-2.* +# httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line); +# httpd-2.4.4/server/protocol.c: "request failed: URI too long (longer than %d)", +# # Author: Tim Connors diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index d40c1c4f..69e5fd49 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -2,3 +2,6 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 +# failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } +[Fri Jul 30 11:23:54 2010] [error] [client 10.85.6.69] request failed: URI too long (longer than 8190) From a4718eb64402d2329bc3891c68024e7d1c61277f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:38:02 +1100 Subject: [PATCH 28/36] ENH: apache-overflow filter to have HTTP-2.4 message IDs and test samples --- config/filter.d/apache-overflows.conf | 19 +++++++++++++++++-- testcases/files/logs/apache-overflows | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 68669222..92551525 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,14 +8,29 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long \(longer than \d+\)|erroneous characters after protocol string) +failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)$ ignoreregex = -# DEV Noptes: +# DEV Notes: # # fgrep -r 'URI too long' httpd-2.* # httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line); # httpd-2.4.4/server/protocol.c: "request failed: URI too long (longer than %d)", # +# fgrep -r 'in request' ../httpd-2.* | fgrep Invalid +# httpd-2.2.25/server/core.c: "Invalid URI in request %s", r->the_request); +# httpd-2.2.25/server/core.c: "Invalid method in request %s", r->the_request); +# httpd-2.2.25/docs/manual/rewrite/flags.html.fr:avertissements 'Invalid URI in request'. +# httpd-2.4.4/server/core.c: "Invalid URI in request %s", r->the_request); +# httpd-2.4.4/server/core.c: "Invalid method in request %s - possible attempt to establish SSL connection on non-SSL port", r->the_request); +# httpd-2.4.4/server/core.c: "Invalid method in request %s", r->the_request); +# +# fgrep -r 'invalid characters in URI' httpd-2.* +# httpd-2.4.4/server/protocol.c: "request failed: invalid characters in URI"); +# +# http://svn.apache.org/viewvc/httpd/httpd/trunk/server/core.c?r1=739382&r2=739620&pathrev=739620 +# ...possible attempt to establish SSL connection on non-SSL port +# +# https://wiki.apache.org/httpd/ListOfErrors # Author: Tim Connors diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 69e5fd49..01f54c7d 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,7 +1,25 @@ +# http://osdir.com/ml/debian-bugs-dist/2010-03/msg05840.html # failJSON: { "time": "2010-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 + # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } [Fri Jul 30 11:23:54 2010] [error] [client 10.85.6.69] request failed: URI too long (longer than 8190) +# failJSON: { "time": "2010-10-27T23:16:37", "match": true , "host": "187.117.240.164" } +[Wed Oct 27 23:16:37 2010] [error] [client 187.117.240.164] Invalid URI in request x\xb2\xa1:SMl\xcc{\xfd"\xd1\x91\x84!d\x0e~\xf6:\xfbVu\xdf\xc3\xdb[\xa9\xfe\xd3lpz\x92\xbf\x9f5\xa3\xbbvF\xbc\xee\x1a\xb1\xb0\xf8K\xecE\xbc\xe8r\xacx=\xc7>\xb5\xbd\xa3\xda\xe9\xf09\x95"fd\x1c\x05\x1c\xd5\xf3#:\x91\xe6WE\xdb\xadN;k14;\xdcr\xad\x9e\xa8\xde\x95\xc3\xebw\xa0\xb1N\x8c~\xf1\xcfSY\xd5zX\xd7\x0f\vH\xe4\xb5(\xcf,3\xc98\x19\xefYq@\xd2I\x96\xfb\xc7\xa9\xae._{S\xd1\x9c\xad\x17\xdci\x9b\xca\x93\xafSM\xb8\x99\xd9|\xc2\xd8\xc9\xe7\xe9O\x99\xad\x19\xc3V]\xcc\xddR\xf7$\xaa\xb8\x18\xe0f\xb8\xff + + +# Could be apache-2.2 or earlier +# http://www.aota.net/forums/showthread.php?t=15796 +# failJSON: { "time": "2003-11-14T16:11:55", "match": true , "host": "1.2.3.4" } +[Fri Nov 14 16:11:55 2003] [error] [client 1.2.3.4] request failed: erroneous characters after protocol string: User-Agent: Mozilla/5.0 (Windows; U; Win98; en-US; m18) Gecko/20001108 Netscape6/6.0 + +# http://forum.directadmin.com/showthread.php?t=22412 +# failJSON: { "time": "2007-11-15T03:09:59", "match": true , "host": "89.189.71.87" } +[Thu Nov 15 03:09:59 2007] [error] [client 89.189.71.87] Invalid method in request NOOP + +# https://issues.apache.org/bugzilla/show_bug.cgi?id=46123 +# failJSON: { "time": "2008-10-29T11:55:14", "match": true , "host": "127.0.0.1" } +[Wed Oct 29 11:55:14 2008] [error] [client 127.0.0.1] Invalid method in request \x16\x03\x01 - possible attempt to establish SSL connection when the server isn't expecting it From c81ed538056c2a9ed163de2354fbd93de731b98c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:40:12 +1100 Subject: [PATCH 29/36] TST: change source URL --- testcases/files/logs/apache-overflows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 01f54c7d..376114c4 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,4 +1,4 @@ -# http://osdir.com/ml/debian-bugs-dist/2010-03/msg05840.html +# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=574182 # failJSON: { "time": "2010-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } From 648d48c35516b5781b36dfa504a77764456dcbfe Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:49:11 +1100 Subject: [PATCH 30/36] ENH: apache-2.4 message IDs for filter apache-noscript --- config/filter.d/apache-noscript.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 4ecf349a..f3c6246a 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -9,10 +9,16 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (File does not exist|script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)\s*$ +failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)\s*$ ^%(_apache_error_client)s script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ ignoreregex = +# DEV Notes: +# +# https://wiki.apache.org/httpd/ListOfErrors for apache error IDs +# +# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is Before http-2.2 +# # Author: Cyril Jaquier From eb9663eb4fd8248c7a6ce82d04edabb3a701c798 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 09:22:41 +1100 Subject: [PATCH 31/36] BF/ENH: asterisk connection ID is a hex not decimal number. Add "Rejecting unknown SIP connection from " regex thanks to Jonathan Lanning --- ChangeLog | 3 +++ THANKS | 1 + config/filter.d/asterisk.conf | 3 ++- testcases/files/logs/asterisk | 3 +++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 27e598ef..50b8bb8e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,9 @@ some obscure corner of the Internet. those filters were used. - Fixes: + Jonathan Lanning + * filter.d/asterisk -- identified another regex for blocking. Also channel + ID is hex not decimal as noted in sample logs provided. Daniel Black & Marcel Dopita * filter.d/apache-auth -- fixed and apache auth samples provide. Closes gh-286 Yaroslav Halchenko diff --git a/THANKS b/THANKS index 5f6d1b2b..13303c21 100644 --- a/THANKS +++ b/THANKS @@ -34,6 +34,7 @@ Guillaume Delvit Hanno 'Rince' Wagner Iain Lea Jonathan Kamens +Jonathan Lanning Jonathan Underwood Joël Bertrand JP Espinosa diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index f77a1557..3c1a97df 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -6,7 +6,7 @@ __pid_re = (?:\[\d+\]) # All Asterisk log messages begin like this: -log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d* +log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d* failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|No matching peer found|Username/auth name mismatch|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ ^%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ @@ -16,6 +16,7 @@ failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' ^%(log_prefix)s Failed to authenticate (user|device) [^@]+@\S*$ ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S*$ ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"(,Challenge="\w+",ReceivedChallenge="\w+")?(,ReceivedHash="[\da-f]+")?$ + ^\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? Ext\. s: "Rejecting unknown SIP connection from "$ ignoreregex = diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index b2eb7738..60c89d5f 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -40,3 +40,6 @@ [2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 # failJSON: { "time": "2013-07-06T09:09:25", "match": true , "host": "141.255.164.106" } [2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39" + +# failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "192.168.55.152" } +[2013-11-11 14:33:38] WARNING[6756][C-0000001d] Ext. s: "Rejecting unknown SIP connection from 192.168.55.152" From c272573fe3318b18cd76d0655481d4fe04b6d9e1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:06:16 +1100 Subject: [PATCH 32/36] ENH: DoS resistant dropbear filter --- config/filter.d/dropbear.conf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dropbear.conf b/config/filter.d/dropbear.conf index 54d8166b..288b0882 100644 --- a/config/filter.d/dropbear.conf +++ b/config/filter.d/dropbear.conf @@ -23,8 +23,8 @@ before = common.conf _daemon = dropbear -failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from :.*$ - ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from .*$ +failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from :\d+$ + ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from (:\d+)?$ ^%(__prefix_line)s[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from :\d+\s*$ ignoreregex = @@ -37,5 +37,12 @@ ignoreregex = # # The second last failregex line we need to match with the modified dropbear. # +# For the second regex the following apply: +# +# http://www.netmite.com/android/mydroid/external/dropbear/svr-authpam.c +# http://svn.dd-wrt.com/changeset/16642#file64 +# +# http://svn.dd-wrt.com/changeset/16642/src/router/dropbear/svr-authpasswd.c +# # Author: Francis Russell # Zak B. Elep From 52972164a273963ded6f0a244deab2f938e49cd4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:13:35 +1100 Subject: [PATCH 33/36] BF: exim filter to be DoS resistant --- config/filter.d/exim.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 5f786594..66743390 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -16,7 +16,7 @@ before = exim-common.conf failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s (plain|login) authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$ - ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ + ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ ignoreregex = @@ -24,5 +24,9 @@ ignoreregex = # DEV Notes: # The %(host_info) defination contains a match # +# SMTP protocol synchronization error \([^)]*\) <- This needs to be non-greedy +# to void capture beyond ")" to avoid a DoS Injection vulnerabilty as input= is +# user injectable data. +# # Author: Cyril Jaquier # Daniel Black (rewrote with strong regexs) From be60518218cc859af4e7ee57e4d22beab27205d5 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:57:01 +1100 Subject: [PATCH 34/36] BF/ENH: DoS resistant roundcube-auth with test cases and more variation in IMAP error given --- config/filter.d/roundcube-auth.conf | 19 ++++++++++++++++--- testcases/files/logs/roundcube-auth | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 279c5edd..b093f69c 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -9,8 +9,21 @@ before = common.conf [Definition] -failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ +failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. .* in .*?/rcube_imap\.php on line \d+ \(\S+ \S+\))?$ ignoreregex = - -# Author: Teodor Micu & Yaroslav Halchenko & terence namusonge +# DEV Notes: +# +# Source: https://github.com/roundcube/roundcubemail/blob/master/program/lib/Roundcube/rcube_imap.php#L180 +# +# Part after comes straight from IMAP server up until the " in ....." +# Earlier versions didn't log the IMAP response hence optional. +# +# DoS resistance: +# +# Assume that the user can inject "from " into the imap response +# somehow. Write test cases around this to ensure that the combination of +# arbitary user input and IMAP response doesn't inject the wrong IP for +# fail2ban +# +# Author: Teodor Micu & Yaroslav Halchenko & terence namusonge & Daniel Black diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index 7c16efbd..43a42192 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -4,3 +4,21 @@ May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 # failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) + +# Made up to attempts to inject a DoS on the server. Assume the user can manipulate the IMAP error response +# +# user = admin from 127.0.0.1 +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# +# IMAP server logs user=${username} +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# +# Old roundcube version - no IMAP response +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4 +# +# user = admin from 127.0.0.1 in +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 in from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) From d0498bec691ab31a8b0968b873c3d11b24609224 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 13 Nov 2013 08:05:08 +1100 Subject: [PATCH 35/36] DOC: finalise 0.8.11 release --- ChangeLog | 10 ++++++++-- MANIFEST | 3 +++ README.md | 2 +- common/version.py | 2 +- man/fail2ban-client.1 | 6 +++--- man/fail2ban-regex.1 | 4 ++-- man/fail2ban-server.1 | 6 +++--- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 50b8bb8e..e901d9af 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,10 +4,10 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.11.pre1) 2013/10/30 +Fail2Ban (version 0.8.11) 2013/11/13 ================================================================================ -ver. 0.8.11 (2013/11/XXX) - loves-unittests-and-tight-DoS-free-filter-regexes +ver. 0.8.11 (2013/11/13) - loves-unittests-and-tight-DoS-free-filter-regexes ----------- In light of CVE-2013-2178 that triggered our last release we have put @@ -23,6 +23,12 @@ please provide us with example log lines on the github issue tracker http://github.com/fail2ban/fail2ban/issues and NOT on a random blog in some obscure corner of the Internet. +Many thanks to our contributors for this release Daniel Black, Yaroslav +Halchenko, Steven Hiscocks, Mark McKinstry, Andy Fragen, Orion Poplawski, +Alexander Dietrich, JP Espinosa, Jamyn Shanley, Beau Raines, François +Boulogne and others who have helped on IRC and mailing list, logged issues +and bug requests. + - IMPORTANT incompatible changes: Filter name changes: * 'lighttpd-fastcgi' filter has been renamed to 'suhosin' diff --git a/MANIFEST b/MANIFEST index 43927a7a..0e0eb327 100644 --- a/MANIFEST +++ b/MANIFEST @@ -60,6 +60,7 @@ testcases/files/logs/exim testcases/files/logs/suhosin testcases/files/logs/mysqld-auth testcases/files/logs/named-refused +testcases/files/logs/nginx-http-auth testcases/files/logs/pam-generic testcases/files/logs/postfix testcases/files/logs/proftpd @@ -139,6 +140,7 @@ config/filter.d/apache-badbots.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf +config/filter.d/nginx-http-auth.conf config/filter.d/courierlogin.conf config/filter.d/couriersmtp.conf config/filter.d/cyrus-imap.conf @@ -239,3 +241,4 @@ files/bash-completion files/fail2ban-tmpfiles.conf files/fail2ban.service files/ipmasq-ZZZzzz_fail2ban.rul +files/gen_badbots diff --git a/README.md b/README.md index 7c00233f..24ed11ff 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.8.11-pre1 2013/10/30 + v0.8.11 2013/11/13 ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/common/version.py b/common/version.py index a0cb94ea..c699a8db 100644 --- a/common/version.py +++ b/common/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -version = "0.8.11.pre1" +version = "0.8.11" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index aab2dde4..c5ccb803 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-CLIENT "1" "October 2013" "fail2ban-client v0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-CLIENT "1" "November 2013" "fail2ban-client v0.8.11" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fIOPTIONS\fR] \fI\fR .SH DESCRIPTION -Fail2Ban v0.8.11.pre1 reads log file that contains password failure report +Fail2Ban v0.8.11 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 4a0e272b..e2c99565 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-REGEX "1" "October 2013" "fail2ban-regex 0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-REGEX "1" "November 2013" "fail2ban-regex 0.8.11" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 7e09b49e..147bdeaa 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-SERVER "1" "October 2013" "fail2ban-server v0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-SERVER "1" "November 2013" "fail2ban-server v0.8.11" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fIOPTIONS\fR] .SH DESCRIPTION -Fail2Ban v0.8.11.pre1 reads log file that contains password failure report +Fail2Ban v0.8.11 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with From 752ea054db514c0d7a47f86d7673fc63edaa304f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 13 Nov 2013 09:01:52 +1100 Subject: [PATCH 36/36] DOC: post release version change --- DEVELOP | 10 +++++----- common/version.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DEVELOP b/DEVELOP index d776d8f7..500f119f 100644 --- a/DEVELOP +++ b/DEVELOP @@ -770,23 +770,23 @@ Look for errors like: Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory - tar -C /tmp -jxf dist/fail2ban-0.8.11.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.8.12.tar.bz2 # clean up current direcory - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.8.11/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.8.12/ # Only differences should be files that you don't want distributed. # Ensure the tests work from the tarball - cd /tmp/fail2ban-0.8.11/ && ./fail2ban-testcases-all + cd /tmp/fail2ban-0.8.12/ && ./fail2ban-testcases-all # Add/finalize the corresponding entry in the ChangeLog To generate a list of committers use e.g. - git shortlog -sn 0.8.10.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.8.11.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' Ensure the top of the ChangeLog has the right version and current date. @@ -857,7 +857,7 @@ Post Release Add the following to the top of the ChangeLog -ver. 0.8.12 (2013/XX/XXX) - wanna-be-released +ver. 0.8.13 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: diff --git a/common/version.py b/common/version.py index c699a8db..eec5e794 100644 --- a/common/version.py +++ b/common/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -version = "0.8.11" +version = "0.8.11.dev"