From 16d63434ef3f4691ee68553582705b2aa7d1ae69 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 11 Jun 2013 23:56:09 +1000 Subject: [PATCH 01/24] DOC: credits --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/3proxy.conf | 17 +++++++++++++++++ config/jail.conf | 6 ++++++ testcases/files/logs/3proxy | 2 ++ 5 files changed, 28 insertions(+) create mode 100644 config/filter.d/3proxy.conf create mode 100644 testcases/files/logs/3proxy diff --git a/ChangeLog b/ChangeLog index e58bce06..d2bf5d42 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.10 (2013/XX/XXX) - NOT-YET-RELEASED * action.d/{route,shorewall}.conf - blocktype must be defined within [Init]. Closes gh-232 - New Features + Daniel Black and ykimon + * filter.d/3proxy filter added - Enhancements Yaroslav Halchenko * jail.conf -- assure all jails have actions and remove unused diff --git a/THANKS b/THANKS index ba33b766..9e151406 100644 --- a/THANKS +++ b/THANKS @@ -48,5 +48,6 @@ Tyler Vaclav Misek Vincent Deffontaines Yaroslav Halchenko +ykimon Yehuda Katz zugeschmiert diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf new file mode 100644 index 00000000..f68ad44e --- /dev/null +++ b/config/filter.d/3proxy.conf @@ -0,0 +1,17 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# +# Requested by ykimon in https://github.com/fail2ban/fail2ban/issues/246 +# + +[Definition] + +# Option: failregex +# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS that 1-9 are all authentication problems +# Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" +# Values: TEXT +# +failregex = \S+\s0000[1-9]\s\S+\s:[0-9]+\S+[0-9]+\s[0-9]+\s.*$ + +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index d3a23920..69197f5b 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -401,4 +401,10 @@ action = pf logpath = /var/log/sshd.log maxretry=5 +[3proxy] + +enabled = false +filter = 3proxy +action = iptables-multiport[name=3proxy, port=318, protocol=tcp] +logpath = /var/log/3proxy.log diff --git a/testcases/files/logs/3proxy b/testcases/files/logs/3proxy new file mode 100644 index 00000000..ff4774f3 --- /dev/null +++ b/testcases/files/logs/3proxy @@ -0,0 +1,2 @@ +11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 From f2fa4d53a8d5fbaa5e4b3769eed8a21832379eb1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 12 Jun 2013 08:30:59 +1000 Subject: [PATCH 02/24] ENH: stricter regex thanks to Steven Hiscocks (kwirk) --- config/filter.d/3proxy.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf index f68ad44e..be3a68d0 100644 --- a/config/filter.d/3proxy.conf +++ b/config/filter.d/3proxy.conf @@ -8,10 +8,10 @@ [Definition] # Option: failregex -# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS that 1-9 are all authentication problems +# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS that 1-9 are all authentication problems (%E field) # Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" # Values: TEXT # -failregex = \S+\s0000[1-9]\s\S+\s:[0-9]+\S+[0-9]+\s[0-9]+\s.*$ +failregex = \s[+-][0-9]{4} \S+ 0000[1-9] \S+ :[0-9]+ [0-9.]+:[0-9]+ [0-9]+ .*$ ignoreregex = From fd9f9f16e0b6ed8ff2c6f9761c1f42c61468e656 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 12 Jun 2013 08:48:30 +1000 Subject: [PATCH 03/24] BF: need to anchor the start to avoid another repeat of DoS injection like Apache --- config/filter.d/3proxy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf index be3a68d0..09855d75 100644 --- a/config/filter.d/3proxy.conf +++ b/config/filter.d/3proxy.conf @@ -12,6 +12,6 @@ # Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" # Values: TEXT # -failregex = \s[+-][0-9]{4} \S+ 0000[1-9] \S+ :[0-9]+ [0-9.]+:[0-9]+ [0-9]+ .*$ +failregex = ^\s[+-][0-9]{4} \S+ 0000[1-9] \S+ :[0-9]+ [0-9.]+:[0-9]+ [0-9]+ .*$ ignoreregex = From 8faf84b7f7540830d40b3c643c8123a9234342a5 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 08:34:10 +1000 Subject: [PATCH 04/24] BF: authentication errors end in 01-09 but the beginning part indicates the service as per https://github.com/fail2ban/fail2ban/issues/246#issuecomment-19327955 thanks to ykimon --- config/filter.d/3proxy.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf index 09855d75..c341cf80 100644 --- a/config/filter.d/3proxy.conf +++ b/config/filter.d/3proxy.conf @@ -8,10 +8,11 @@ [Definition] # Option: failregex -# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS that 1-9 are all authentication problems (%E field) +# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are +# all authentication problems (%E field) # Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" # Values: TEXT # -failregex = ^\s[+-][0-9]{4} \S+ 0000[1-9] \S+ :[0-9]+ [0-9.]+:[0-9]+ [0-9]+ .*$ +failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ :\d+ [0-9.]+:\d+ \d+ ignoreregex = From 9dbaec0894ff64de9e902771da9ccae9d2c8b7bf Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 10:23:14 +1000 Subject: [PATCH 05/24] ENH: sample log + more specific regex --- config/filter.d/3proxy.conf | 2 +- testcases/files/logs/3proxy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf index c341cf80..b22b4588 100644 --- a/config/filter.d/3proxy.conf +++ b/config/filter.d/3proxy.conf @@ -13,6 +13,6 @@ # Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" # Values: TEXT # -failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ :\d+ [0-9.]+:\d+ \d+ +failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ :\d+ [\d.]+:\d+ \d+ \d+ \d+\s ignoreregex = diff --git a/testcases/files/logs/3proxy b/testcases/files/logs/3proxy index ff4774f3..2967c9bf 100644 --- a/testcases/files/logs/3proxy +++ b/testcases/files/logs/3proxy @@ -1,2 +1,3 @@ 11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +13-06-2013 01:39:34 +0300 PROXY.3128 00508 - 1.2.3.4:28938 0.0.0.0:0 0 0 0 From 88b4598ed8a59ee5984b3e9a556ab873894811c3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 14:43:15 +1000 Subject: [PATCH 06/24] BF: fix to proxy port in 3proxy example --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 69197f5b..c999cc7b 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -405,6 +405,6 @@ maxretry=5 enabled = false filter = 3proxy -action = iptables-multiport[name=3proxy, port=318, protocol=tcp] +action = iptables-multiport[name=3proxy, port=3128, protocol=tcp] logpath = /var/log/3proxy.log From 3e3802512aa1d18c3749aac11122c237508bad75 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 17:44:18 +1000 Subject: [PATCH 07/24] ENH/BF: exim improvements with sample --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/exim.conf | 2 +- testcases/files/logs/exim | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 590edfa6..6bdcd9e1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - New Features: - Enhancements: + Daniel Black and Georgiy Mernov + * filter.d/exim -- update regex for new exim fail log message with sample ver. 0.8.10 (2013/06/12) - wanna-be-secure diff --git a/THANKS b/THANKS index ba33b766..05c46561 100644 --- a/THANKS +++ b/THANKS @@ -18,6 +18,7 @@ Daniel Black David Nutter Eric Gerbier Enrico Labedzki +Georgiy Mernov Guillaume Delvit Hanno 'Rince' Wagner Iain Lea diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index b846e992..511f30b8 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -14,7 +14,7 @@ # Values: TEXT # failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) - login authenticator failed for .* \[\]: 535 Incorrect authentication data \(set_id=.*\)\s*$ + ^ login authenticator failed for ([^\s\(]+\s)?\([^)]*\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index dd507379..5bfc4060 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -1,2 +1,4 @@ # From IRC 2013-01-04 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) +# From IRC 2013-06-13 XATRIX (Georgiy Mernov) +2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) From 4c67a269bf9d3186201c72305cead1820fc1fbaa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 22:11:05 +1000 Subject: [PATCH 08/24] ENH: proftp regex hardening and log messages --- ChangeLog | 2 ++ config/filter.d/proftpd.conf | 17 ++++++++++++----- testcases/files/logs/proftpd | 3 +++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 590edfa6..ef5ab555 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - New Features: - Enhancements: + Daniel Black + * filter.d/proftpd -- harden and anchor regex and add two testcases ver. 0.8.10 (2013/06/12) - wanna-be-secure diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index f28e2d4b..3a16b877 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -1,8 +1,15 @@ # Fail2Ban configuration file # # Author: Yaroslav Halchenko +# Daniel Black - hardening of regex # -# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] @@ -13,10 +20,10 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = \(\S+\[\]\)[: -]+ USER \S+: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - \(\S+\[\]\)[: -]+ USER \S+ \(Login failed\): .*$ - \(\S+\[\]\)[: -]+ SECURITY VIOLATION: \S+ login attempted\. *$ - \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ +failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER \S+: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER \S+ \(Login failed\): .*$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: \S+ login attempted\. *$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index def8a83e..5c88c1c1 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -1,5 +1,8 @@ Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 +Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. +Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21 +Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. From dbe7ffe050057ef6ea022127110222908bf2673d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Jun 2013 23:52:15 +1000 Subject: [PATCH 09/24] ENH: dovecot regexs rewritten and extra failures --- ChangeLog | 3 +++ config/filter.d/dovecot.conf | 21 ++++++++++++--------- testcases/files/logs/dovecot | 8 +++++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 590edfa6..240fe494 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - New Features: - Enhancements: + Daniel Black + * filter/dovecot -- updated to use full anchored regex with extra failure + examples in sample logs ver. 0.8.10 (2013/06/12) - wanna-be-secure diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index d7fb6e6d..dd4c35ba 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -1,20 +1,23 @@ -# Fail2Ban configuration file for dovcot +# Fail2Ban configuration file for dovecot # # Author: Martin Waschbuesch -# -# +# Daniel Black (rewrote with begin and end anchors) + +[INCLUDES] + +before = common.conf [Definition] +_daemon = dovecot(-auth)? + # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Notes.: regex to match the password failures messages in the logfile. +# first regex is essentially a copy of pam-generic.conf # Values: TEXT # -failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*\s+rip=(?P\S*),.* - pam.*dovecot.*(?:authentication failure).*\s+rhost=(?:\s+user=.*)?\s*$ +failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3},( TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 434acade..fd84789e 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -3,6 +3,12 @@ @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root +@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 + +Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS +Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected + From 2e2ec5d1f515cb6dc02eb0d40c665fc6b992fb34 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 00:17:41 +1000 Subject: [PATCH 10/24] ENH: injection of fail data into USER field --- testcases/files/logs/proftpd | 1 + 1 file changed, 1 insertion(+) diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index 5c88c1c1..aaaf0295 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -3,6 +3,7 @@ Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123. Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21 Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. +Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 From 9940cd1b6b0146c2a088edab611e8d77e1d2984d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 00:29:43 +1000 Subject: [PATCH 11/24] ENH: proftpd chan accept usernames with spaces --- config/filter.d/proftpd.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 3a16b877..13080fcc 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -20,9 +20,9 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER \S+: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER \S+ \(Login failed\): .*$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: \S+ login attempted\. *$ +failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex From e8b6acfa6521f0995adaf06db9349b260d438544 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 00:53:03 +1000 Subject: [PATCH 12/24] TST: attempts at injection with username=rhost=1.2.3.4 have no user= logged in dovecot-1.2.15 --- testcases/files/logs/dovecot | 1 + 1 file changed, 1 insertion(+) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index fd84789e..f904a8fe 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -8,6 +8,7 @@ Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS +Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected From 6a09ecff5c8c1115919b825b822d343c41061da3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 08:41:50 +1000 Subject: [PATCH 13/24] ENH: anchor a bit mor. Use \d and \w where possible. Escape a literal . --- config/filter.d/asterisk.conf | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index c1b3dcab..56ea68b4 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -14,25 +14,22 @@ before = common.conf [Definition] # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P\S+) +# Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Wrong password$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - No matching peer found$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Username/auth name mismatch$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Device does not match ACL$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Peer is not supposed to register$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - ACL error \(permit/deny\)$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Not a local domain$ - NOTICE%(__pid_re)s\[[^:]+\] [^:]+: Call from '[^']*' \(:[0-9]+\) to extension '[0-9]+' rejected because extension not found in context 'default'.$ - NOTICE%(__pid_re)s [^:]+: Host failed to authenticate as '[^']*'$ - NOTICE%(__pid_re)s [^:]+: No registration for peer '[^']*' \(from \)$ - NOTICE%(__pid_re)s [^:]+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ - NOTICE%(__pid_re)s [^:]+: Failed to authenticate user [^@]+@\S*$ - SECURITY%(__pid_re)s [^:]+: SecurityEvent="InvalidAccountID",EventTV="[0-9-]+",Severity="[a-zA-Z]+",Service="[a-zA-Z]+",EventVersion="[0-9]+",AccountID="[0-9]+",SessionID="0x[0-9a-f]+",LocalAddress="IPV[46]/(UD|TC)P/[0-9a-fA-F:.]+/[0-9]+",RemoteAddress="IPV[46]/(UD|TC)P//[0-9]+"$ +failregex = \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ + \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ + \]\s*NOTICE%(__pid_re)s\[[^:]+\] [^:]+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + \]\s*NOTICE%(__pid_re)s [^:]+: Host failed to authenticate as '[^']*'$ + \]\s*NOTICE%(__pid_re)s [^:]+: No registration for peer '[^']*' \(from \)$ + \]\s*NOTICE%(__pid_re)s [^:]+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ + \]\s*NOTICE%(__pid_re)s [^:]+: Failed to authenticate user [^@]+@\S*$ + \]\s*SECURITY%(__pid_re)s [^:]+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From d4940563d3f3be6eae68c0ff4ebd869e2145356f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 08:55:25 +1000 Subject: [PATCH 14/24] ENH: regex hardening on assp --- ChangeLog | 2 ++ config/filter.d/assp.conf | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 590edfa6..a9e02d68 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - New Features: - Enhancements: + Daniel Black + * filter.d/assp -- regex hardening ver. 0.8.10 (2013/06/12) - wanna-be-secure diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index b1bfc082..4b8f5caf 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -20,9 +20,9 @@ # Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded __assp_actions = (dropping|refusing) -failregex = max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: [a-zA-Z0-9]+;$ - SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ - Blocking - too much AUTH errors \(\d{,3}\);$ +failregex = ^ \[SSL-out\] max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ + ^ \[SSL-out\] SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ + ^ Blocking - too much AUTH errors \(\d{,3}\);$ # Option: ignoreregex From a447aa615d2d6114db4af582576836dea9ce78b2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 12:27:35 +1000 Subject: [PATCH 15/24] BF: [SSL-out] is optional in assp --- config/filter.d/assp.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 4b8f5caf..2b98ffc2 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -18,10 +18,10 @@ # Examples: Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); # Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; # Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded -__assp_actions = (dropping|refusing) +__assp_actions = (?:dropping|refusing) -failregex = ^ \[SSL-out\] max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ - ^ \[SSL-out\] SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ +failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ + ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ ^ Blocking - too much AUTH errors \(\d{,3}\);$ From 7018d81244d560d6fc0d40418f7331b8dfd609e1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 12:35:44 +1000 Subject: [PATCH 16/24] BF: missed a space --- config/filter.d/assp.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 2b98ffc2..2854d898 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -20,7 +20,7 @@ # Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded __assp_actions = (?:dropping|refusing) -failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ +failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ ^ Blocking - too much AUTH errors \(\d{,3}\);$ From 09302c5c2566b6924bcb1243b86d031ad5398e6b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 18:58:13 -0400 Subject: [PATCH 17/24] ENH: asterisk -- use \S instead of [^:] + prefix failregex with ^\[ detected date portion is stripped from the string to be matched, so it is not only the right ] is left, but also the left one ;-) --- config/filter.d/asterisk.conf | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 56ea68b4..a8f65e09 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -17,19 +17,19 @@ before = common.conf # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ - \]\s*NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ - \]\s*NOTICE%(__pid_re)s\[[^:]+\] [^:]+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ - \]\s*NOTICE%(__pid_re)s [^:]+: Host failed to authenticate as '[^']*'$ - \]\s*NOTICE%(__pid_re)s [^:]+: No registration for peer '[^']*' \(from \)$ - \]\s*NOTICE%(__pid_re)s [^:]+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ - \]\s*NOTICE%(__pid_re)s [^:]+: Failed to authenticate user [^@]+@\S*$ - \]\s*SECURITY%(__pid_re)s [^:]+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ +failregex = ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ + ^\[\]\s*NOTICE%(__pid_re)s\[\S+\] \S+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed to authenticate as '[^']*'$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: No registration for peer '[^']*' \(from \)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Failed to authenticate user [^@]+@\S*$ + ^\[\]\s*SECURITY%(__pid_re)s \S+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 97f9cfc0b05cd00ada984eed6514b96546fb30a1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 22:56:36 -0400 Subject: [PATCH 18/24] ENH: 'heavydebug' level == 5 for even more debugging in tricky cases I mocked logging library directly -- seems to be Ok. --- common/__init__.py | 5 +++++ fail2ban-testcases | 5 +++-- server/filter.py | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/common/__init__.py b/common/__init__.py index 2b76f4b6..3eae8ee3 100644 --- a/common/__init__.py +++ b/common/__init__.py @@ -23,3 +23,8 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" + +import logging + +# Custom debug level +logging.HEAVYDEBUG = 5 diff --git a/fail2ban-testcases b/fail2ban-testcases index 0bd70259..fc5cc146 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -52,7 +52,7 @@ def get_opt_parser(): p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), default=None, help="Log level for the logger to use during running tests"), Option('-n', "--no-network", action="store_true", @@ -76,7 +76,8 @@ parser = get_opt_parser() logSys = logging.getLogger("fail2ban") # Numerical level of verbosity corresponding to a log "level" -verbosity = {'debug': 3, +verbosity = {'heavydebug': 4, + 'debug': 3, 'info': 2, 'warn': 1, 'error': 1, diff --git a/server/filter.py b/server/filter.py index 90530f92..59e612ed 100644 --- a/server/filter.py +++ b/server/filter.py @@ -291,6 +291,8 @@ class Filter(JailThread): except UnicodeDecodeError: l = line l = l.rstrip('\r\n') + + logSys.log(5, "Working on line %r", l) timeMatch = self.dateDetector.matchTime(l) if timeMatch: # Lets split into time part and log part of the line @@ -355,6 +357,8 @@ class Filter(JailThread): if failRegex.hasMatched(): # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) + logSys.log(7, "Date: %r, message: %r", + timeLine, logLine) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" From ffe381d91cbe92cf505df93648fd322f5673155d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 22:19:10 -0400 Subject: [PATCH 19/24] RF: reworked -regex cmdline tool to use optparse, some unification and enhancement of outputs --- fail2ban-regex | 464 +++++++++++++++++++++++++------------------------ 1 file changed, 237 insertions(+), 227 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index a0a90b05..af38d4d9 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -17,9 +17,17 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +Report bugs to https://github.com/fail2ban/fail2ban/issues +""" __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" import getopt, sys, time, logging, os @@ -37,202 +45,201 @@ from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderErro from server.filter import Filter from server.failregex import RegexException +from optparse import OptionParser, Option + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.regex") -class RegexStat: +def shortstr(s, l=53): + """Return shortened string + """ + if len(s) > l: + return s[:l-3] + '...' + return s + +def pprint_list(l, header=None): + if not len(l): + return + if header: + s = "|- %s\n" % header + else: + s = '' + print s + "| " + "\n| ".join(l) + '\n`-' + +def get_opt_parser(): + # use module docstring for help output + p = OptionParser( + usage="%s [OPTIONS] [IGNOREREGEX]\n" % sys.argv[0] + __doc__ + + """ +LOG: + string a string representing a log line + filename path to a log file (/var/log/auth.log) + +REGEX: + string a string representing a 'failregex' + filename path to a filter file (filter.d/sshd.conf) + +IGNOREREGEX: + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +""", + version="%prog " + version) + + p.add_options([ + Option('-l', "--log-level", type="choice", + dest="log_level", + choices=('debug', 'info', 'warn', 'error', 'fatal'), + default=None, + help="Log level for the Fail2Ban logger to use"), + Option("-v", "--verbose", action='store_true', + help="Be verbose in output"), + Option("--print-all-missed", action='store_true', + help="Either to print all missed lines"), + Option("--print-all-ignored", action='store_true', + help="Either to print all ignored lines"), + + ]) + + return p + + +class RegexStat(object): def __init__(self, failregex): - self.__stats = 0 - self.__failregex = failregex - self.__ipList = list() + self._stats = 0 + self._failregex = failregex + self._ipList = list() def __str__(self): return "%s(%r) %d failed: %s" \ - % (self.__class__, self.__failregex, self.__stats, self.__ipList) + % (self.__class__, self._failregex, self._stats, self._ipList) def inc(self): - self.__stats += 1 + self._stats += 1 def getStats(self): - return self.__stats + return self._stats def getFailRegex(self): - return self.__failregex + return self._failregex def appendIP(self, value): - self.__ipList.extend(value) + self._ipList.extend(value) def getIPList(self): - return self.__ipList + return self._ipList -class Fail2banRegex: +class LineStats(object): + """Just a convenience container for stats + """ + def __init__(self): + self.tested = self.matched = 0 + self.missed_lines = [] + self.ignored_lines = [] + + def __str__(self): + return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self + + @property + def ignored(self): + return len(self.ignored_lines) + + @property + def missed(self): + return self.tested - (self.ignored + self.matched) + + # just for convenient str + def __getitem__(self, key): + return getattr(self, key) + + +class Fail2banRegex(object): + + # ??? test = None CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} - def __init__(self): - self.__filter = Filter(None) - self.__ignoreregex = list() - self.__failregex = list() - self.__verbose = False + def __init__(self, opts): + self._verbose = opts.verbose + self._print_all_missed = opts.print_all_missed + self._print_all_ignored = opts.print_all_ignored + + self._filter = Filter(None) + self._ignoreregex = list() + self._failregex = list() + self._line_stats = LineStats() + # Setup logging logging.getLogger("fail2ban").handlers = [] - self.__hdlr = logging.StreamHandler(Fail2banRegex.test) + self._hdlr = logging.StreamHandler(Fail2banRegex.test) # set a format which is simpler for console use formatter = logging.Formatter("%(message)s") # tell the handler to use this format - self.__hdlr.setFormatter(formatter) - self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self.__hdlr) + self._hdlr.setFormatter(formatter) + self._logging_level = self._verbose and logging.DEBUG or logging.WARN + logging.getLogger("fail2ban").addHandler(self._hdlr) logging.getLogger("fail2ban").setLevel(logging.ERROR) - #@staticmethod - def dispVersion(): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - dispVersion = staticmethod(dispVersion) - #@staticmethod - def dispUsage(): - print "Usage: "+sys.argv[0]+" [OPTIONS] [IGNOREREGEX]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "This tools can test regular expressions for \"fail2ban\"." - print - print "Options:" - print " -h, --help display this help message" - print " -V, --version print the version" - print " -v, --verbose verbose output" - print - print "Log:" - print " string a string representing a log line" - print " filename path to a log file (/var/log/auth.log)" - print - print "Regex:" - print " string a string representing a 'failregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "IgnoreRegex:" - print " string a string representing an 'ignoreregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" - dispUsage = staticmethod(dispUsage) - - def getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) - elif opt[0] in ["-v", "--verbose"]: - self.__verbose = True - - #@staticmethod - def logIsFile(value): - return os.path.isfile(value) - logIsFile = staticmethod(logIsFile) - - def readIgnoreRegex(self, value): + def readRegex(self, value, regextype): + assert(regextype in ('fail', 'ignore')) + regex = regextype + 'regex' if os.path.isfile(value): reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) try: reader.read(value) - print "Use ignoreregex file : " + value - self.__ignoreregex = [RegexStat(m) - for m in reader.get("Definition", "ignoreregex").split('\n')] + print "Use %11s file : %s" % (regex, value) + # TODO: reuse functionality in client + regex_values = [RegexStat(m) + for m in reader.get("Definition", regex).split('\n')] except NoSectionError: - print "No [Definition] section in " + value - print + print "No [Definition] section in %s" % value return False except NoOptionError: - print "No failregex option in " + value - print + print "No %s option in %s" % (regex, value) return False except MissingSectionHeaderError: - print "No section headers in " + value - print + print "No section headers in %s" % value return False else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use ignoreregex line : " + stripReg - self.__ignoreregex = [RegexStat(value)] - return True + print "Use %11s line : %s" % (regex, shortstr(value)) + regex_values = [RegexStat(value)] - def readRegex(self, value): - if os.path.isfile(value): - reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) - try: - reader.read(value) - print "Use regex file : " + value - self.__failregex = [RegexStat(m) - for m in reader.get("Definition", "failregex").split('\n')] - except NoSectionError: - print "No [Definition] section in " + value - print - return False - except NoOptionError: - print "No failregex option in " + value - print - return False - except MissingSectionHeaderError: - print "No section headers in " + value - print - return False - else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use regex line : " + stripReg - self.__failregex = [RegexStat(value)] + setattr(self, "_" + regex, regex_values) return True def testIgnoreRegex(self, line): found = False - for regex in self.__ignoreregex: - logging.getLogger("fail2ban").setLevel(self.__logging_level) + for regex in self._ignoreregex: + logging.getLogger("fail2ban").setLevel(self._logging_level) try: - self.__filter.addIgnoreRegex(regex.getFailRegex()) + self._filter.addIgnoreRegex(regex.getFailRegex()) try: - ret = self.__filter.ignoreLine(line) + ret = self._filter.ignoreLine(line) if ret: + found = True regex.inc() except RegexException, e: print e return False finally: - self.__filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self.__logging_level) + self._filter.delIgnoreRegex(0) + logging.getLogger("fail2ban").setLevel(self._logging_level) + return found def testRegex(self, line): found = False - for regex in self.__ignoreregex: - self.__filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self.__failregex: - logging.getLogger("fail2ban").setLevel(logging.DEBUG) + for regex in self._ignoreregex: + self._filter.addIgnoreRegex(regex.getFailRegex()) + for regex in self._failregex: + # logging.getLogger("fail2ban").setLevel(logging.DEBUG) try: - self.__filter.addFailRegex(regex.getFailRegex()) + self._filter.addFailRegex(regex.getFailRegex()) try: - ret = self.__filter.processLine(line) - if not len(ret) == 0: + ret = self._filter.processLine(line) + if len(ret): if found == True: ret[0].append(True) else: @@ -247,16 +254,47 @@ class Fail2banRegex: print "Sorry, but no found in regex" return False finally: - self.__filter.delFailRegex(0) + self._filter.delFailRegex(0) logging.getLogger("fail2ban").setLevel(logging.CRITICAL) - for regex in self.__ignoreregex: - self.__filter.delIgnoreRegex(0) + for regex in self._ignoreregex: + self._filter.delIgnoreRegex(0) + return found + + + def process(self, test_lines): + + for line in test_lines: + if line.startswith('# ') or not line.strip(): + # skip comment and empty lines + continue + is_ignored = fail2banRegex.testIgnoreRegex(line) + if is_ignored: + self._line_stats.ignored_lines.append(line) + + if fail2banRegex.testRegex(line): + assert(not is_ignored) + self._line_stats.matched += 1 + else: + if not is_ignored: + self._line_stats.missed_lines.append(line) + self._line_stats.tested += 1 + + def printLines(self, ltype): + lstats = self._line_stats + assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) + l = lstats[ltype + '_lines'] + if len(l): + header = "%s line(s):" % (ltype.capitalize(),) + if len(l) < 20 or getattr(self, '_print_all_' + ltype): + pprint_list([x.rstrip() for x in l], header) + else: + print "%s: too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, len(l)) def printStats(self): print print "Results" print "=======" - print def print_failregexes(title, failregexes): # Print title @@ -264,106 +302,78 @@ class Fail2banRegex: for cnt, failregex in enumerate(failregexes): match = failregex.getStats() total += match - if (match or self.__verbose): - out.append("| %d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - print "%s: %d total" % (title, total) - if len(out): - print "|- #) [# of hits] regular expression" - print '\n'.join(out) - print '`-' - print - return total + if (match or self._verbose): + out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - # Print title - total = print_failregexes("Failregex", self.__failregex) - _ = print_failregexes("Ignoreregex", self.__ignoreregex) - - print "Summary" - print "=======" - print - - if total == 0: - print "Sorry, no match" - print - print "Look at the above section 'Running tests' which could contain important" - print "information." - return False - else: - # Print stats - print "Addresses found:" - for cnt, failregex in enumerate(self.__failregex): - if self.__verbose or len(failregex.getIPList()): - print "[%d]" % (cnt+1) + if self._verbose and len(failregex.getIPList()): for ip in failregex.getIPList(): timeTuple = time.localtime(ip[1]) timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - print " %s (%s)%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "") - print + out.append(" %s %s%s" % ( + ip[0], timeString, ip[2] and " (already matched)" or "")) - print "Date template hits:" - for template in self.__filter.dateDetector.getTemplates(): - if self.__verbose or template.getHits(): - print `template.getHits()` + " hit(s): " + template.getName() - print + print "\n%s: %d total" % (title, total) + pprint_list(out, " #) [# of hits] regular expression") + return total - print "Success, the total number of match is " + str(total) - print - print "However, look at the above section 'Running tests' which could contain important" - print "information." - return True + # Print title + total = print_failregexes("Failregex", self._failregex) + _ = print_failregexes("Ignoreregex", self._ignoreregex) + + + print "\nDate template hits:" + out = [] + for template in self._filter.dateDetector.getTemplates(): + if self._verbose or template.getHits(): + out.append("[%d] %s" % (template.getHits(), template.getName())) + pprint_list(out, "[# of hits] date format") + + print "\nLines: %s" % self._line_stats + + self.printLines('ignored') + self.printLines('missed') + + return True if __name__ == "__main__": - fail2banRegex = Fail2banRegex() - # Reads the command line options. - try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] - optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - fail2banRegex.dispUsage() - sys.exit(-1) - # Process command line - fail2banRegex.getCmdLineOptions(optList) + + parser = get_opt_parser() + (opts, args) = parser.parse_args() + + fail2banRegex = Fail2banRegex(opts) # We need 2 or 3 parameters if not len(args) in (2, 3): - fail2banRegex.dispUsage() + sys.stderr.write("ERROR: provide both and .\n\n") + parser.print_help() sys.exit(-1) + + print + print "Running tests" + print "=============" + print + + cmd_log, cmd_regex = args[:2] + + if len(args) == 3: + fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) + + fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log) + print "Use log file : %s" % cmd_log + test_lines = hdlr.readlines() + except IOError, e: + print e + sys.exit(-1) else: - print - print "Running tests" - print "=============" - print + print "Use single line : %s" % shortstr(cmd_log) + test_lines = [ cmd_log ] + print - cmd_log, cmd_regex = args[:2] + fail2banRegex.process(test_lines) - if len(args) == 3: - fail2banRegex.readIgnoreRegex(args[2]) or sys.exit(-1) - - fail2banRegex.readRegex(cmd_regex) or sys.exit(-1) - - if fail2banRegex.logIsFile(cmd_log): - try: - hdlr = open(cmd_log) - print "Use log file : " + cmd_log - print - for line in hdlr: - fail2banRegex.testIgnoreRegex(line) - fail2banRegex.testRegex(line) - except IOError, e: - print e - print - sys.exit(-1) - else: - if len(sys.argv[1]) > 53: - stripLog = cmd_log[0:50] + "..." - else: - stripLog = cmd_log - print "Use single line: " + stripLog - print - fail2banRegex.testIgnoreRegex(cmd_log) - fail2banRegex.testRegex(cmd_log) - - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex.printStats() or sys.exit(-1) From e91419d361c9120896b67c92a1b811a7edf467ef Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 23:01:35 -0400 Subject: [PATCH 20/24] ENH: fail2ban-regex -- add specification of loglevels to enable --- fail2ban-regex | 60 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index af38d4d9..4f5ffd73 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -40,15 +40,16 @@ except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") from common.version import version +from optparse import OptionParser, Option + from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException -from optparse import OptionParser, Option - +from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.regex") +logSys = logging.getLogger("fail2ban") def shortstr(s, l=53): """Return shortened string @@ -88,7 +89,7 @@ IGNOREREGEX: p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), default=None, help="Log level for the Fail2Ban logger to use"), Option("-v", "--verbose", action='store_true', @@ -97,6 +98,10 @@ IGNOREREGEX: help="Either to print all missed lines"), Option("--print-all-ignored", action='store_true', help="Either to print all ignored lines"), + Option("-t", "--log-traceback", action='store_true', + help="Enrich log-messages with compressed tracebacks"), + Option("--full-traceback", action='store_true', + help="Either to make the tracebacks full, not compressed (as by default)"), ]) @@ -156,9 +161,6 @@ class LineStats(object): class Fail2banRegex(object): - # ??? - test = None - CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} def __init__(self, opts): @@ -171,17 +173,6 @@ class Fail2banRegex(object): self._failregex = list() self._line_stats = LineStats() - # Setup logging - logging.getLogger("fail2ban").handlers = [] - self._hdlr = logging.StreamHandler(Fail2banRegex.test) - # set a format which is simpler for console use - formatter = logging.Formatter("%(message)s") - # tell the handler to use this format - self._hdlr.setFormatter(formatter) - self._logging_level = self._verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self._hdlr) - logging.getLogger("fail2ban").setLevel(logging.ERROR) - def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) @@ -213,7 +204,6 @@ class Fail2banRegex(object): def testIgnoreRegex(self, line): found = False for regex in self._ignoreregex: - logging.getLogger("fail2ban").setLevel(self._logging_level) try: self._filter.addIgnoreRegex(regex.getFailRegex()) try: @@ -226,7 +216,6 @@ class Fail2banRegex(object): return False finally: self._filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self._logging_level) return found def testRegex(self, line): @@ -234,7 +223,6 @@ class Fail2banRegex(object): for regex in self._ignoreregex: self._filter.addIgnoreRegex(regex.getFailRegex()) for regex in self._failregex: - # logging.getLogger("fail2ban").setLevel(logging.DEBUG) try: self._filter.addFailRegex(regex.getFailRegex()) try: @@ -255,7 +243,6 @@ class Fail2banRegex(object): return False finally: self._filter.delFailRegex(0) - logging.getLogger("fail2ban").setLevel(logging.CRITICAL) for regex in self._ignoreregex: self._filter.delIgnoreRegex(0) return found @@ -349,6 +336,35 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) + # TODO: taken from -testcases -- move common functionality somewhere + if opts.log_level is not None: # pragma: no cover + # so we had explicit settings + logSys.setLevel(getattr(logging, opts.log_level.upper())) + else: # pragma: no cover + # suppress the logging but it would leave unittests' progress dots + # ticking, unless like with '-l fatal' which would be silent + # unless error occurs + logSys.setLevel(getattr(logging, 'FATAL')) + + # Add the default logging handler + stdout = logging.StreamHandler(sys.stdout) + + fmt = 'D: %(message)s' + + if opts.log_traceback: + Formatter = FormatterWithTraceBack + fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt + else: + Formatter = logging.Formatter + + # Custom log format for the verbose tests runs + if opts.verbose > 1: # pragma: no cover + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) + else: # pragma: no cover + # just prefix with the space + stdout.setFormatter(Formatter(fmt)) + logSys.addHandler(stdout) + print print "Running tests" print "=============" From 9b351350dd5c5b6e91a2eabc31741d95fffdf63c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 23:19:08 -0400 Subject: [PATCH 21/24] DOC: Changelog for asterisk hardening --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 590edfa6..1ea46aed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,7 +15,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - New Features: - Enhancements: - + Daniel Black + * config/filter.d/asterisk.conf -- more stringent anchoring ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From 77044fce35be6e7e1d6310521227f6a25ea61ac9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 23:21:48 -0400 Subject: [PATCH 22/24] DOC: Changelog for fail2ban-regex RF --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1ea46aed..316d20f2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - Enhancements: Daniel Black * config/filter.d/asterisk.conf -- more stringent anchoring + Yaroslav Halchenko + * fail2ban-regex -- refactored to provide more details (missing and + ignored lines, control over logging, etc) while maintaining look&feel. ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From a433a8ea5fa4762d46208977327428329c62fcc3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 15:21:50 +1000 Subject: [PATCH 23/24] ENH: readibility thanks to Yaroslav --- config/filter.d/exim.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 511f30b8..2c4a516b 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -14,7 +14,7 @@ # Values: TEXT # failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) - ^ login authenticator failed for ([^\s\(]+\s)?\([^)]*\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))\s*$ + ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 8cc13b5b4013137a51365013c3bd36618fff2548 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Jun 2013 18:12:53 +1000 Subject: [PATCH 24/24] BF/ENH: Incorrect authentication data doesn't need tailier so that's optional. Also gained log entry for Unrouteable address --- config/filter.d/exim.conf | 4 ++-- testcases/files/logs/exim | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 2c4a516b..d335c9fc 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -13,8 +13,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) - ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))\s*$ +failregex = ^ H=\S+ \(\S+\) \[\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$ + ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 5bfc4060..d86b6455 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -2,3 +2,5 @@ 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) # From IRC 2013-06-13 XATRIX (Georgiy Mernov) 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) +2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data +2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address