From ac061155f093464fb6cd2329d3d513b15c68e256 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 8 Nov 2013 14:40:52 -0800 Subject: [PATCH 01/15] BF: anchor introduced nginx-http-auth at the end needed since request probably could be not a correct HTTP statement but continue with all those to match till the end and then injected ", client: VICTIM, server..." thus allowing injection. We better anchor at the end then --- config/filter.d/nginx-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 00f152b7..79dda30b 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+" +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"\s*$ ignoreregex = From 724c6bfd922638ecc1eddf9197df5369eeb7cd0b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 9 Nov 2013 10:35:13 +1100 Subject: [PATCH 02/15] DOC: filter regex debugging --- DEVELOP | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/DEVELOP b/DEVELOP index 9a3be94b..0e37a0c3 100644 --- a/DEVELOP +++ b/DEVELOP @@ -289,15 +289,19 @@ TIP: Some applications log spaces at the end. If you are not sure add \s*$ as the end part of the regex. If your regex is not matching, http://www.debuggex.com/?flavor=python can help -to tune it: +to tune it. fail2ban-regex -D ... will present Debuggex URLs for the regexs +and sample log files that you pass into it. +In general use when using regex debuggers for generating fail2ban filters: * use regex from the ./fail2ban-regex output (to ensure all substitutions are -done) and replace with (?&.ipv4). Make sure that regex type set to -Python; -* for the test data put your log output with the time removed; -- when you have fixed the regex put it back into your filter file. +done) +* replace with (?&.ipv4) +* make sure that regex type set to Python +* for the test data put your log output with the date/time removed -Please spread the good word about debuggex - Serge Toarca is kindly continuing +When you have fixed the regex put it back into your filter file. + +Please spread the good word about Debuggex - Serge Toarca is kindly continuing its free availability to Open Source developers. Finishing up: From b8f40fef1bf362f5645b912b651797e4d2b83de2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:08:10 +1100 Subject: [PATCH 03/15] DOC: more on filter regexes - DEVELOP --- DEVELOP | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/DEVELOP b/DEVELOP index 0e37a0c3..d776d8f7 100644 --- a/DEVELOP +++ b/DEVELOP @@ -331,7 +331,7 @@ failregex, while matching inserted text to the part, they have the ability to deny any host they choose. So the part must be anchored on text generated by the application, and -not the user, to a extent sufficient to prevent user inserting the entire text +not the user, to an extent sufficient to prevent user inserting the entire text matching this or any other failregex. Ideally filter regex should anchor at the beginning and at the end of log line. @@ -381,7 +381,7 @@ Note if we'd just had the expression: Then provided the user put a space in their command they would have never been banned. -2. Filter regex can match other user injected data +2. Unanchored regex can match other user injected data From the Apache vulnerability CVE-2013-2178 ( original ref: https://vndh.net/note:fail2ban-089-denial-service ). @@ -402,7 +402,82 @@ Now the log line will be: As this log line doesn't match other expressions hence it matches the above regex and blocks 192.168.33.1 as a denial of service from the HTTP requester. -3. Application generates two identical log messages with different meanings +3. Over greedy pattern matching + +From: https://github.com/fail2ban/fail2ban/pull/426 + +An example ssh log (simplified) + + Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser remoteuser + +As we assume username can include anything including spaces its prudent to put +.* here. The remote user can also exist as anything so lets not make assumptions again. + + failregex = ^%(__prefix_line)sFailed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + +So this works. The problem is if the .* after remote user is injected by the +user to be 'from 1.2.3.4'. The resultant log line is. + + Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 + +Testing with: + + fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$' + +TIP: I've removed the bit that matches __prefix_line from the regex and log. + +Shows: + + 1) [1] ^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + 1.2.3.4 Sun Sep 29 17:15:02 2013 + +It should of matched 127.0.0.1. So the first greedy part of the greedy regex +matched until the end of the string. The was no "from " so the regex +engine worked backwards from the end of the string until this was matched. + +The result was that 1.2.3.4 was matched, injected by the user, and the wrong IP +was banned. + +The solution here is to make the first .* non-greedy with .*?. Here it matches +as little as required and the fail2ban-regex tool shows the output: + + fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$' + + 1) [1] ^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$ + 127.0.0.1 Sun Sep 29 17:15:02 2013 + +So the general case here is a log line that contains: + + (fixed_data_1)(fixed_data_2)(user_injectable_data) + +Where the regex that matches fixed_data_1 is gready and matches the entire +string, before moving backwards and user_injectable_data can match the entire +string. + +Another case: + +ref: https://www.debuggex.com/r/CtAbeKMa2sDBEfA2/0 + +A webserver logs the following without URL escaping: + + [error] 2865#0: *66647 user "xyz" was not found in "/file", client: 1.2.3.1, server: www.host.com, request: "GET ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host", host: "www.myhost.com" + +regex: + + failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (?:password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ .+ HTTP/\d+\.\d+", host: "\S+" + +The .* matches to the end of the string. Finds that it can't continue to match +", client ... so it moves from the back and find that the user injected web URL: + + ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host + +In this case there is a fixed host: "www.myhost.com" at the end so the solution +is to anchor the regex at the end with a $. + +If this wasn't the case then first .* needed to be made so it didn't capture +beyond . + +4. Application generates two identical log messages with different meanings If the application generates the following two messages under different circumstances: From d955714d26ba78f11999b4bd99a7c36ff8c55679 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:11:32 +1100 Subject: [PATCH 04/15] TST: test case that shows injection --- testcases/files/logs/sshd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 4f862d89..ed4857bd 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -100,3 +100,6 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 + +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From 061a26c40815b8594e9ffce1e9056aeac7dff5cd Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:28:09 +1100 Subject: [PATCH 05/15] TST: fix space in sshd sample log --- testcases/files/logs/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index ed4857bd..541afb19 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -101,5 +101,5 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 -# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From d90130234dfba1fd389118941dcf4ec8db031c8b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 08:29:54 +1100 Subject: [PATCH 06/15] TST: end of json in sshd sample log --- testcases/files/logs/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 541afb19..3c50dcfd 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -101,5 +101,5 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 -# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" +# failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From 87516eb92b5792ce1202a5d67da481f6019ad52a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 09:46:40 +1100 Subject: [PATCH 07/15] ENH: apache-overflows - more detail on "request failed: URI too long (longer than %d)" with test case --- config/filter.d/apache-overflows.conf | 8 +++++++- testcases/files/logs/apache-overflows | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index de1c770d..68669222 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,8 +8,14 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long|erroneous characters after protocol string) +failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long \(longer than \d+\)|erroneous characters after protocol string) ignoreregex = +# DEV Noptes: +# +# fgrep -r 'URI too long' httpd-2.* +# httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line); +# httpd-2.4.4/server/protocol.c: "request failed: URI too long (longer than %d)", +# # Author: Tim Connors diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index d40c1c4f..69e5fd49 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -2,3 +2,6 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 +# failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } +[Fri Jul 30 11:23:54 2010] [error] [client 10.85.6.69] request failed: URI too long (longer than 8190) From a4718eb64402d2329bc3891c68024e7d1c61277f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:38:02 +1100 Subject: [PATCH 08/15] ENH: apache-overflow filter to have HTTP-2.4 message IDs and test samples --- config/filter.d/apache-overflows.conf | 19 +++++++++++++++++-- testcases/files/logs/apache-overflows | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 68669222..92551525 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,14 +8,29 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (Invalid (method|URI) in request|request failed: URI too long \(longer than \d+\)|erroneous characters after protocol string) +failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)$ ignoreregex = -# DEV Noptes: +# DEV Notes: # # fgrep -r 'URI too long' httpd-2.* # httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line); # httpd-2.4.4/server/protocol.c: "request failed: URI too long (longer than %d)", # +# fgrep -r 'in request' ../httpd-2.* | fgrep Invalid +# httpd-2.2.25/server/core.c: "Invalid URI in request %s", r->the_request); +# httpd-2.2.25/server/core.c: "Invalid method in request %s", r->the_request); +# httpd-2.2.25/docs/manual/rewrite/flags.html.fr:avertissements 'Invalid URI in request'. +# httpd-2.4.4/server/core.c: "Invalid URI in request %s", r->the_request); +# httpd-2.4.4/server/core.c: "Invalid method in request %s - possible attempt to establish SSL connection on non-SSL port", r->the_request); +# httpd-2.4.4/server/core.c: "Invalid method in request %s", r->the_request); +# +# fgrep -r 'invalid characters in URI' httpd-2.* +# httpd-2.4.4/server/protocol.c: "request failed: invalid characters in URI"); +# +# http://svn.apache.org/viewvc/httpd/httpd/trunk/server/core.c?r1=739382&r2=739620&pathrev=739620 +# ...possible attempt to establish SSL connection on non-SSL port +# +# https://wiki.apache.org/httpd/ListOfErrors # Author: Tim Connors diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 69e5fd49..01f54c7d 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,7 +1,25 @@ +# http://osdir.com/ml/debian-bugs-dist/2010-03/msg05840.html # failJSON: { "time": "2010-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 + # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } [Fri Jul 30 11:23:54 2010] [error] [client 10.85.6.69] request failed: URI too long (longer than 8190) +# failJSON: { "time": "2010-10-27T23:16:37", "match": true , "host": "187.117.240.164" } +[Wed Oct 27 23:16:37 2010] [error] [client 187.117.240.164] Invalid URI in request x\xb2\xa1:SMl\xcc{\xfd"\xd1\x91\x84!d\x0e~\xf6:\xfbVu\xdf\xc3\xdb[\xa9\xfe\xd3lpz\x92\xbf\x9f5\xa3\xbbvF\xbc\xee\x1a\xb1\xb0\xf8K\xecE\xbc\xe8r\xacx=\xc7>\xb5\xbd\xa3\xda\xe9\xf09\x95"fd\x1c\x05\x1c\xd5\xf3#:\x91\xe6WE\xdb\xadN;k14;\xdcr\xad\x9e\xa8\xde\x95\xc3\xebw\xa0\xb1N\x8c~\xf1\xcfSY\xd5zX\xd7\x0f\vH\xe4\xb5(\xcf,3\xc98\x19\xefYq@\xd2I\x96\xfb\xc7\xa9\xae._{S\xd1\x9c\xad\x17\xdci\x9b\xca\x93\xafSM\xb8\x99\xd9|\xc2\xd8\xc9\xe7\xe9O\x99\xad\x19\xc3V]\xcc\xddR\xf7$\xaa\xb8\x18\xe0f\xb8\xff + + +# Could be apache-2.2 or earlier +# http://www.aota.net/forums/showthread.php?t=15796 +# failJSON: { "time": "2003-11-14T16:11:55", "match": true , "host": "1.2.3.4" } +[Fri Nov 14 16:11:55 2003] [error] [client 1.2.3.4] request failed: erroneous characters after protocol string: User-Agent: Mozilla/5.0 (Windows; U; Win98; en-US; m18) Gecko/20001108 Netscape6/6.0 + +# http://forum.directadmin.com/showthread.php?t=22412 +# failJSON: { "time": "2007-11-15T03:09:59", "match": true , "host": "89.189.71.87" } +[Thu Nov 15 03:09:59 2007] [error] [client 89.189.71.87] Invalid method in request NOOP + +# https://issues.apache.org/bugzilla/show_bug.cgi?id=46123 +# failJSON: { "time": "2008-10-29T11:55:14", "match": true , "host": "127.0.0.1" } +[Wed Oct 29 11:55:14 2008] [error] [client 127.0.0.1] Invalid method in request \x16\x03\x01 - possible attempt to establish SSL connection when the server isn't expecting it From c81ed538056c2a9ed163de2354fbd93de731b98c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:40:12 +1100 Subject: [PATCH 09/15] TST: change source URL --- testcases/files/logs/apache-overflows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 01f54c7d..376114c4 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,4 +1,4 @@ -# http://osdir.com/ml/debian-bugs-dist/2010-03/msg05840.html +# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=574182 # failJSON: { "time": "2010-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } From 648d48c35516b5781b36dfa504a77764456dcbfe Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 11 Nov 2013 10:49:11 +1100 Subject: [PATCH 10/15] ENH: apache-2.4 message IDs for filter apache-noscript --- config/filter.d/apache-noscript.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 4ecf349a..f3c6246a 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -9,10 +9,16 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (File does not exist|script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)\s*$ +failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)\s*$ ^%(_apache_error_client)s script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ ignoreregex = +# DEV Notes: +# +# https://wiki.apache.org/httpd/ListOfErrors for apache error IDs +# +# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is Before http-2.2 +# # Author: Cyril Jaquier From eb9663eb4fd8248c7a6ce82d04edabb3a701c798 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 09:22:41 +1100 Subject: [PATCH 11/15] BF/ENH: asterisk connection ID is a hex not decimal number. Add "Rejecting unknown SIP connection from " regex thanks to Jonathan Lanning --- ChangeLog | 3 +++ THANKS | 1 + config/filter.d/asterisk.conf | 3 ++- testcases/files/logs/asterisk | 3 +++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 27e598ef..50b8bb8e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,9 @@ some obscure corner of the Internet. those filters were used. - Fixes: + Jonathan Lanning + * filter.d/asterisk -- identified another regex for blocking. Also channel + ID is hex not decimal as noted in sample logs provided. Daniel Black & Marcel Dopita * filter.d/apache-auth -- fixed and apache auth samples provide. Closes gh-286 Yaroslav Halchenko diff --git a/THANKS b/THANKS index 5f6d1b2b..13303c21 100644 --- a/THANKS +++ b/THANKS @@ -34,6 +34,7 @@ Guillaume Delvit Hanno 'Rince' Wagner Iain Lea Jonathan Kamens +Jonathan Lanning Jonathan Underwood Joël Bertrand JP Espinosa diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index f77a1557..3c1a97df 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -6,7 +6,7 @@ __pid_re = (?:\[\d+\]) # All Asterisk log messages begin like this: -log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d* +log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d* failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|No matching peer found|Username/auth name mismatch|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ ^%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ @@ -16,6 +16,7 @@ failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' ^%(log_prefix)s Failed to authenticate (user|device) [^@]+@\S*$ ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S*$ ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"(,Challenge="\w+",ReceivedChallenge="\w+")?(,ReceivedHash="[\da-f]+")?$ + ^\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? Ext\. s: "Rejecting unknown SIP connection from "$ ignoreregex = diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index b2eb7738..60c89d5f 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -40,3 +40,6 @@ [2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 # failJSON: { "time": "2013-07-06T09:09:25", "match": true , "host": "141.255.164.106" } [2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39" + +# failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "192.168.55.152" } +[2013-11-11 14:33:38] WARNING[6756][C-0000001d] Ext. s: "Rejecting unknown SIP connection from 192.168.55.152" From c272573fe3318b18cd76d0655481d4fe04b6d9e1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:06:16 +1100 Subject: [PATCH 12/15] ENH: DoS resistant dropbear filter --- config/filter.d/dropbear.conf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dropbear.conf b/config/filter.d/dropbear.conf index 54d8166b..288b0882 100644 --- a/config/filter.d/dropbear.conf +++ b/config/filter.d/dropbear.conf @@ -23,8 +23,8 @@ before = common.conf _daemon = dropbear -failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from :.*$ - ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from .*$ +failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from :\d+$ + ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from (:\d+)?$ ^%(__prefix_line)s[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from :\d+\s*$ ignoreregex = @@ -37,5 +37,12 @@ ignoreregex = # # The second last failregex line we need to match with the modified dropbear. # +# For the second regex the following apply: +# +# http://www.netmite.com/android/mydroid/external/dropbear/svr-authpam.c +# http://svn.dd-wrt.com/changeset/16642#file64 +# +# http://svn.dd-wrt.com/changeset/16642/src/router/dropbear/svr-authpasswd.c +# # Author: Francis Russell # Zak B. Elep From 52972164a273963ded6f0a244deab2f938e49cd4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:13:35 +1100 Subject: [PATCH 13/15] BF: exim filter to be DoS resistant --- config/filter.d/exim.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 5f786594..66743390 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -16,7 +16,7 @@ before = exim-common.conf failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s (plain|login) authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$ - ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ + ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ ignoreregex = @@ -24,5 +24,9 @@ ignoreregex = # DEV Notes: # The %(host_info) defination contains a match # +# SMTP protocol synchronization error \([^)]*\) <- This needs to be non-greedy +# to void capture beyond ")" to avoid a DoS Injection vulnerabilty as input= is +# user injectable data. +# # Author: Cyril Jaquier # Daniel Black (rewrote with strong regexs) From be60518218cc859af4e7ee57e4d22beab27205d5 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 12 Nov 2013 18:57:01 +1100 Subject: [PATCH 14/15] BF/ENH: DoS resistant roundcube-auth with test cases and more variation in IMAP error given --- config/filter.d/roundcube-auth.conf | 19 ++++++++++++++++--- testcases/files/logs/roundcube-auth | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 279c5edd..b093f69c 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -9,8 +9,21 @@ before = common.conf [Definition] -failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ +failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. .* in .*?/rcube_imap\.php on line \d+ \(\S+ \S+\))?$ ignoreregex = - -# Author: Teodor Micu & Yaroslav Halchenko & terence namusonge +# DEV Notes: +# +# Source: https://github.com/roundcube/roundcubemail/blob/master/program/lib/Roundcube/rcube_imap.php#L180 +# +# Part after comes straight from IMAP server up until the " in ....." +# Earlier versions didn't log the IMAP response hence optional. +# +# DoS resistance: +# +# Assume that the user can inject "from " into the imap response +# somehow. Write test cases around this to ensure that the combination of +# arbitary user input and IMAP response doesn't inject the wrong IP for +# fail2ban +# +# Author: Teodor Micu & Yaroslav Halchenko & terence namusonge & Daniel Black diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index 7c16efbd..43a42192 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -4,3 +4,21 @@ May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 # failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) + +# Made up to attempts to inject a DoS on the server. Assume the user can manipulate the IMAP error response +# +# user = admin from 127.0.0.1 +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# +# IMAP server logs user=${username} +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# +# Old roundcube version - no IMAP response +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4 +# +# user = admin from 127.0.0.1 in +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 in from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) From d0498bec691ab31a8b0968b873c3d11b24609224 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 13 Nov 2013 08:05:08 +1100 Subject: [PATCH 15/15] DOC: finalise 0.8.11 release --- ChangeLog | 10 ++++++++-- MANIFEST | 3 +++ README.md | 2 +- common/version.py | 2 +- man/fail2ban-client.1 | 6 +++--- man/fail2ban-regex.1 | 4 ++-- man/fail2ban-server.1 | 6 +++--- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 50b8bb8e..e901d9af 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,10 +4,10 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.11.pre1) 2013/10/30 +Fail2Ban (version 0.8.11) 2013/11/13 ================================================================================ -ver. 0.8.11 (2013/11/XXX) - loves-unittests-and-tight-DoS-free-filter-regexes +ver. 0.8.11 (2013/11/13) - loves-unittests-and-tight-DoS-free-filter-regexes ----------- In light of CVE-2013-2178 that triggered our last release we have put @@ -23,6 +23,12 @@ please provide us with example log lines on the github issue tracker http://github.com/fail2ban/fail2ban/issues and NOT on a random blog in some obscure corner of the Internet. +Many thanks to our contributors for this release Daniel Black, Yaroslav +Halchenko, Steven Hiscocks, Mark McKinstry, Andy Fragen, Orion Poplawski, +Alexander Dietrich, JP Espinosa, Jamyn Shanley, Beau Raines, François +Boulogne and others who have helped on IRC and mailing list, logged issues +and bug requests. + - IMPORTANT incompatible changes: Filter name changes: * 'lighttpd-fastcgi' filter has been renamed to 'suhosin' diff --git a/MANIFEST b/MANIFEST index 43927a7a..0e0eb327 100644 --- a/MANIFEST +++ b/MANIFEST @@ -60,6 +60,7 @@ testcases/files/logs/exim testcases/files/logs/suhosin testcases/files/logs/mysqld-auth testcases/files/logs/named-refused +testcases/files/logs/nginx-http-auth testcases/files/logs/pam-generic testcases/files/logs/postfix testcases/files/logs/proftpd @@ -139,6 +140,7 @@ config/filter.d/apache-badbots.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf +config/filter.d/nginx-http-auth.conf config/filter.d/courierlogin.conf config/filter.d/couriersmtp.conf config/filter.d/cyrus-imap.conf @@ -239,3 +241,4 @@ files/bash-completion files/fail2ban-tmpfiles.conf files/fail2ban.service files/ipmasq-ZZZzzz_fail2ban.rul +files/gen_badbots diff --git a/README.md b/README.md index 7c00233f..24ed11ff 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.8.11-pre1 2013/10/30 + v0.8.11 2013/11/13 ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/common/version.py b/common/version.py index a0cb94ea..c699a8db 100644 --- a/common/version.py +++ b/common/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -version = "0.8.11.pre1" +version = "0.8.11" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index aab2dde4..c5ccb803 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-CLIENT "1" "October 2013" "fail2ban-client v0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-CLIENT "1" "November 2013" "fail2ban-client v0.8.11" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fIOPTIONS\fR] \fI\fR .SH DESCRIPTION -Fail2Ban v0.8.11.pre1 reads log file that contains password failure report +Fail2Ban v0.8.11 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 4a0e272b..e2c99565 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-REGEX "1" "October 2013" "fail2ban-regex 0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-REGEX "1" "November 2013" "fail2ban-regex 0.8.11" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 7e09b49e..147bdeaa 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.12. -.TH FAIL2BAN-SERVER "1" "October 2013" "fail2ban-server v0.8.11.pre1" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4. +.TH FAIL2BAN-SERVER "1" "November 2013" "fail2ban-server v0.8.11" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fIOPTIONS\fR] .SH DESCRIPTION -Fail2Ban v0.8.11.pre1 reads log file that contains password failure report +Fail2Ban v0.8.11 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with