From 6ce41a611d8f4a7daf8b522ba25aa230f726ca96 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 11 Jul 2013 22:13:51 +1000 Subject: [PATCH 01/60] BF: fix filter on apache-auth. Closes #286 --- ChangeLog | 3 ++- THANKS | 1 + config/filter.d/apache-auth.conf | 8 ++++++-- testcases/files/logs/apache-auth | 5 ++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index f0df7ddc..c289be80 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,7 +11,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released ----------- - Fixes: - + Daniel Black & Marcel Dopita + * filter.d/apache-auth -- fixed and apache auth samples provide. closes #286 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/THANKS b/THANKS index 47c3e999..26e8041f 100644 --- a/THANKS +++ b/THANKS @@ -28,6 +28,7 @@ Joël Bertrand Justin Shore Kévin Drapel kojiro +Marcel Dopita Mark Edgington Markus Hoffmann Marvin Rouge diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index ae3232f2..aa906401 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -18,8 +18,12 @@ before = apache-common.conf # be used for standard IP/hostname matching and is only an alias for # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -# -failregex = ^%(_apache_error_client)s user .* (authentication failure|not found|password mismatch)\s*$ + +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c + +failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ + ^%(_apache_error_client)s user .* not found: \S*\s*$ + # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index cf0f6d30..744a3406 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -2,4 +2,7 @@ [Sat Jun 01 02:17:42 2013] [error] [client 192.168.33.1] File does not exist: /srv/http/site/[client 192.168.0.1] user root not found # should match -[Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found +# from https://github.com/fail2ban/fail2ban/issues/286 +[Thu Jul 11 01:21:41 2013] [error] [client 194.228.20.113] user not found: / +[Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: / +[Thu Jul 11 01:21:45 2013] [error] [client 194.228.20.113] user dsfasdf not found: / From fce431add8a2bc940b7c56bec723c543d167da8a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 11 Jul 2013 22:28:27 +1000 Subject: [PATCH 02/60] ENH: add mod_authz_core failures to apache-auth --- config/filter.d/apache-auth.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index aa906401..e99af22c 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -19,10 +19,14 @@ before = apache-common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c (1st 2) +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_authz_core.c failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ + ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ + ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ + ^%(_apache_error_client)s user .* authorization failure for "\S*": \S*\s*$ # Option: ignoreregex From 384b72a535ada04cbd2062beeabf9ece81dd722e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 11 Jul 2013 22:58:36 +1000 Subject: [PATCH 03/60] ENH: apache-auth filter - client wrong auth --- config/filter.d/apache-auth.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index e99af22c..82aedea0 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -19,11 +19,12 @@ before = apache-common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c (1st 2) -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_authz_core.c +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c (1st 3) +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_authz_core.c (2nd 3) failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ + ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s user .* authorization failure for "\S*": \S*\s*$ From 10e3be857a7d1aec02fea14ed1566bde6297ad22 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 11 Jul 2013 23:08:46 +1000 Subject: [PATCH 04/60] ENH: apache-auth filter added mod_auth_digest message --- config/filter.d/apache-auth.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 82aedea0..0ed7fe34 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -21,6 +21,7 @@ before = apache-common.conf # http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c (1st 3) # http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_authz_core.c (2nd 3) +# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_digest.c (next 4) failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ @@ -28,6 +29,10 @@ failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s user .* authorization failure for "\S*": \S*\s*$ + ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ + ^%(_apache_error_client)s invalid nonce .* received - user attempted time travel\s*$ + ^%(_apache_error_client)s user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ + ^%(_apache_error_client)s user .*: one-time-nonce mismatch - sending new nonce\s*$ # Option: ignoreregex From bf54db7ce711b65aa423099e086ae3e203836076 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 11 Jul 2013 23:10:44 +1000 Subject: [PATCH 05/60] ENH: ChangeLog notes for apache-auth enhancements --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index c289be80..14e8b2ef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening and extra failure examples in sample logs + * filter.d/apache-auth - added expressions for mod_authz, mod_auth and + mod_auth_digest failures. Daniel Black & Georgiy Mernov * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs From 975999591f19f397eccf589089a73579f5ff7ea4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 12 Jul 2013 07:39:18 +1000 Subject: [PATCH 06/60] ENH/DOC: more realm mismatch errors. Documented filter design criteria --- config/filter.d/apache-auth.conf | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 0ed7fe34..366e25ab 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -12,27 +12,37 @@ before = apache-common.conf [Definition] -# Option: failregex -# Notes.: regex to match the password failure messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# Values: TEXT - -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_basic.c (1st 3) -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_authz_core.c (2nd 3) -# http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/mod_auth_digest.c (next 4) +# This filter matches the authorization failures of Apache. It takes the log messages +# from the modules in aaa that return HTTP_UNAUTHORIZED, HTTP_METHOD_NOT_ALLOWED or +# HTTP_FORBIDDEN and not AUTH_GENERAL_ERROR or HTTP_INTERNAL_SERVER_ERROR. +# +# As an unauthorized response is the first step for a browser to instigate authentication +# so don't set max retry too low. +# +# Source: +# +# By searching the code in http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/aaa/* +# for ap_log_rerror(APLOG_MARK, APLOG_ERR and examining resulting return code should get +# all of these expressions. Lots of submodules like mod_authz_* return back to mod_authz_core +# to return the actual failure. failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ - ^%(_apache_error_client)s user .* authorization failure for "\S*": \S*\s*$ + ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ + ^%(_apache_error_client)s user .* authorization failure for "\S*": \s*$ ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ ^%(_apache_error_client)s invalid nonce .* received - user attempted time travel\s*$ ^%(_apache_error_client)s user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ ^%(_apache_error_client)s user .*: one-time-nonce mismatch - sending new nonce\s*$ + ^%(_apache_error_client)s realm mismatch - got `.*' but expected `.+'\s*$ + ^%(_apache_error_client)s unknown algorithm `\S+' received: \S*\s*"$ + ^%(_apache_error_client)s user `.*' in realm `.+' (not found|denied by provider): \S*\s*"$ + ^%(_apache_error_client)s user .* password mismatch: \S*\s*"$ + ^%(_apache_error_client)s invalid qop `.*' received: \S*\s*"$ + # Option: ignoreregex From 1bb427cc14a9faf005270b4b8f34cbced5fa465e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 12 Jul 2013 09:09:24 +1000 Subject: [PATCH 07/60] TST: remove dup test log entry --- testcases/files/logs/apache-auth | 1 - 1 file changed, 1 deletion(-) diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 744a3406..e673e86d 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -5,4 +5,3 @@ # from https://github.com/fail2ban/fail2ban/issues/286 [Thu Jul 11 01:21:41 2013] [error] [client 194.228.20.113] user not found: / [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: / -[Thu Jul 11 01:21:45 2013] [error] [client 194.228.20.113] user dsfasdf not found: / From 40cc336cd565c2140f4680c2c4e6c9d1064f57a7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 17 Jul 2013 22:46:04 +1000 Subject: [PATCH 08/60] TST: testcases and logs for apache-auth basic --- testcases/files/config/apache-auth/README | 13 +++++++++++++ .../files/config/apache-auth/basic/file/.htaccess | 5 +++++ .../files/config/apache-auth/basic/file/.htpasswd | 1 + testcases/files/logs/apache-auth | 15 +++++++++++++++ 4 files changed, 34 insertions(+) create mode 100644 testcases/files/config/apache-auth/README create mode 100644 testcases/files/config/apache-auth/basic/file/.htaccess create mode 100644 testcases/files/config/apache-auth/basic/file/.htpasswd diff --git a/testcases/files/config/apache-auth/README b/testcases/files/config/apache-auth/README new file mode 100644 index 00000000..b25a30b2 --- /dev/null +++ b/testcases/files/config/apache-auth/README @@ -0,0 +1,13 @@ + +Apache Auth. + +This directory contains the configuration file of Apache's Web Server to +simulate authentication files. + +These assumed that /var/www/html is the web root and AllowOverides is "All". + +The subdirectories here are copied to the /var/www/html directory. + +Commands executed are in testcases/files/log/apache-auth with their +corresponding failure mechanism. + diff --git a/testcases/files/config/apache-auth/basic/file/.htaccess b/testcases/files/config/apache-auth/basic/file/.htaccess new file mode 100644 index 00000000..e36e884b --- /dev/null +++ b/testcases/files/config/apache-auth/basic/file/.htaccess @@ -0,0 +1,5 @@ +AuthType basic +AuthName "private area" +AuthBasicProvider file +AuthUserFile /var/www/html/basic/file/.htpasswd +Require valid-user diff --git a/testcases/files/config/apache-auth/basic/file/.htpasswd b/testcases/files/config/apache-auth/basic/file/.htpasswd new file mode 100644 index 00000000..fcc6ec72 --- /dev/null +++ b/testcases/files/config/apache-auth/basic/file/.htpasswd @@ -0,0 +1 @@ +username:$apr1$uUMsOjCQ$.BzXClI/B/vZKddgIAJCR. diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index e673e86d..37fff9eb 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -5,3 +5,18 @@ # from https://github.com/fail2ban/fail2ban/issues/286 [Thu Jul 11 01:21:41 2013] [error] [client 194.228.20.113] user not found: / [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: / + +# The failures below use the configuration described in testcases/conf/apache-auth +# +# wget --http-user=username --http-password=wrongpass http://localhost/basic/file -O /dev/null +# failJSON: { "time": "2013-07-17T22:18:52", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 22:18:52 2013] [error] [client 127.0.0.1] user username: authentication failure for "/basic/file": Password Mismatch + +# wget --http-user=wrongusername --http-password=wrongpass http://localhost/basic/file -O /dev/null +# failJSON: { "time": "2013-07-17T22:32:48", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 22:32:48 2013] [error] [client 127.0.0.1] user wrongusername not found: /basic/file + +# wget --header='Authorization: Digest username="Mufasa",realm="testrealm@host.com",nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",uri="/dir/index.html",qop=auth,nc=00000001,cnonce="0a4f113b",response="6629fae49393a05397450978507c4ef1",opaque="5ccc069c403ebaf9f0171e9517f40e41"' http://localhost/basic/file -O /dev/null +# failJSON: { "time": "2013-07-17T22:39:55", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 22:39:55 2013] [error] [client 127.0.0.1] client used wrong authentication scheme: /basic/file + From e0292913eb69203033397600c09f5ce7f124a756 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 17 Jul 2013 23:05:04 +1000 Subject: [PATCH 09/60] ENH/TST: filter, testcase and log entry for apache-auth authorization scheme mod_authz_owner --- config/filter.d/apache-auth.conf | 1 + .../files/config/apache-auth/basic/authz_owner/.htaccess | 5 +++++ .../files/config/apache-auth/basic/authz_owner/.htpasswd | 1 + .../config/apache-auth/basic/authz_owner/cant_get_me.html | 0 testcases/files/logs/apache-auth | 3 +++ 5 files changed, 10 insertions(+) create mode 100644 testcases/files/config/apache-auth/basic/authz_owner/.htaccess create mode 100644 testcases/files/config/apache-auth/basic/authz_owner/.htpasswd create mode 100644 testcases/files/config/apache-auth/basic/authz_owner/cant_get_me.html diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 366e25ab..34089abb 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -29,6 +29,7 @@ before = apache-common.conf failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ + ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: file owner \S+ does not match.\s*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ diff --git a/testcases/files/config/apache-auth/basic/authz_owner/.htaccess b/testcases/files/config/apache-auth/basic/authz_owner/.htaccess new file mode 100644 index 00000000..583e5422 --- /dev/null +++ b/testcases/files/config/apache-auth/basic/authz_owner/.htaccess @@ -0,0 +1,5 @@ +AuthType basic +AuthName "private area" +AuthBasicProvider file +AuthUserFile /var/www/html/basic/authz_owner/.htpasswd +Require file-owner diff --git a/testcases/files/config/apache-auth/basic/authz_owner/.htpasswd b/testcases/files/config/apache-auth/basic/authz_owner/.htpasswd new file mode 100644 index 00000000..d9de6185 --- /dev/null +++ b/testcases/files/config/apache-auth/basic/authz_owner/.htpasswd @@ -0,0 +1 @@ +username:$apr1$1f5oQUl4$21lLXSN7xQOPtNsj5s4Nk/ diff --git a/testcases/files/config/apache-auth/basic/authz_owner/cant_get_me.html b/testcases/files/config/apache-auth/basic/authz_owner/cant_get_me.html new file mode 100644 index 00000000..e69de29b diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 37fff9eb..2c1b6a73 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -20,3 +20,6 @@ # failJSON: { "time": "2013-07-17T22:39:55", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:39:55 2013] [error] [client 127.0.0.1] client used wrong authentication scheme: /basic/file +# wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null +# failJSON: { "time": "2013-07-17T22:54:32", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 22:54:32 2013] [error] [client 127.0.0.1] Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match. From 4eca2c0bd5d76156266ceb250abd545863c88490 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 17 Jul 2013 23:24:19 +1000 Subject: [PATCH 10/60] TST: apache-auth client denied by server configuration --- config/filter.d/apache-auth.conf | 5 +++-- testcases/files/config/apache-auth/noentry/.htaccess | 1 + testcases/files/logs/apache-auth | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 testcases/files/config/apache-auth/noentry/.htaccess diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 34089abb..47373573 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -26,12 +26,13 @@ before = apache-common.conf # all of these expressions. Lots of submodules like mod_authz_* return back to mod_authz_core # to return the actual failure. -failregex = ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ +failregex = ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ + ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: file owner \S+ does not match.\s*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ - ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ + ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ ^%(_apache_error_client)s user .* authorization failure for "\S*": \s*$ ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ diff --git a/testcases/files/config/apache-auth/noentry/.htaccess b/testcases/files/config/apache-auth/noentry/.htaccess new file mode 100644 index 00000000..3a428827 --- /dev/null +++ b/testcases/files/config/apache-auth/noentry/.htaccess @@ -0,0 +1 @@ +Deny from all diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 2c1b6a73..b84631f7 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -8,6 +8,15 @@ # The failures below use the configuration described in testcases/conf/apache-auth # + +# wget http://localhost/noentry/cant_get_me.html -O /dev/null +# failJSON: { "time": "2013-07-17T23:20:45", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 23:20:45 2013] [error] [client 127.0.0.1] client denied by server configuration: /var/www/html/noentry/cant_get_me.html + +# wget --http-user='' --http-password='' http://localhost/basic/file/cant_get_me.html -O /dev/null +# failJSON: { "time": "2013-07-17T23:14:37", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 23:14:37 2013] [error] [client 127.0.0.1] user not found: /basic/anon/cant_get_me.html + # wget --http-user=username --http-password=wrongpass http://localhost/basic/file -O /dev/null # failJSON: { "time": "2013-07-17T22:18:52", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:18:52 2013] [error] [client 127.0.0.1] user username: authentication failure for "/basic/file": Password Mismatch @@ -23,3 +32,4 @@ # wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-17T22:54:32", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:54:32 2013] [error] [client 127.0.0.1] Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match. + From f8b5b3a1ef831df9a5a539dbf304f8c5b07292ee Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 17 Jul 2013 23:31:44 +1000 Subject: [PATCH 11/60] ENH: apache-auth - quite a lot of authorization failure messages depending on module. Make a wildcard --- config/filter.d/apache-auth.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 47373573..2cc746f0 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -30,9 +30,8 @@ failregex = ^%(_apache_error_client)s client denied by server configuration: (ur ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ - ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: file owner \S+ does not match.\s*$ + ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: .*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ - ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ ^%(_apache_error_client)s user .* authorization failure for "\S*": \s*$ ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ From 8ce9c78474afb94f14c05bf0a5d7d68669f819f4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Jul 2013 00:36:17 +1000 Subject: [PATCH 12/60] TST: apache-auth digest logs --- config/filter.d/apache-auth.conf | 8 +++++--- testcases/files/config/apache-auth/digest/.htaccess | 6 ++++++ testcases/files/config/apache-auth/digest/.htpasswd | 1 + testcases/files/config/apache-auth/digest_anon/.htaccess | 9 +++++++++ testcases/files/config/apache-auth/digest_anon/.htpasswd | 3 +++ .../files/config/apache-auth/digest_wrongrelm/.htaccess | 6 ++++++ .../files/config/apache-auth/digest_wrongrelm/.htpasswd | 2 ++ testcases/files/logs/apache-auth | 7 +++++++ 8 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 testcases/files/config/apache-auth/digest/.htaccess create mode 100644 testcases/files/config/apache-auth/digest/.htpasswd create mode 100644 testcases/files/config/apache-auth/digest_anon/.htaccess create mode 100644 testcases/files/config/apache-auth/digest_anon/.htpasswd create mode 100644 testcases/files/config/apache-auth/digest_wrongrelm/.htaccess create mode 100644 testcases/files/config/apache-auth/digest_wrongrelm/.htpasswd diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 2cc746f0..083cfbc8 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -25,13 +25,17 @@ before = apache-common.conf # for ap_log_rerror(APLOG_MARK, APLOG_ERR and examining resulting return code should get # all of these expressions. Lots of submodules like mod_authz_* return back to mod_authz_core # to return the actual failure. - +# +# See also: http://wiki.apache.org/httpd/ListOfErrors +# failregex = ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: .*$ ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ + ^%(_apache_error_client)s (Digest: )?user .*: password mismatch: \S*\s*$ + ^%(_apache_error_client)s (Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ ^%(_apache_error_client)s user .* authorization failure for "\S*": \s*$ ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ @@ -40,8 +44,6 @@ failregex = ^%(_apache_error_client)s client denied by server configuration: (ur ^%(_apache_error_client)s user .*: one-time-nonce mismatch - sending new nonce\s*$ ^%(_apache_error_client)s realm mismatch - got `.*' but expected `.+'\s*$ ^%(_apache_error_client)s unknown algorithm `\S+' received: \S*\s*"$ - ^%(_apache_error_client)s user `.*' in realm `.+' (not found|denied by provider): \S*\s*"$ - ^%(_apache_error_client)s user .* password mismatch: \S*\s*"$ ^%(_apache_error_client)s invalid qop `.*' received: \S*\s*"$ diff --git a/testcases/files/config/apache-auth/digest/.htaccess b/testcases/files/config/apache-auth/digest/.htaccess new file mode 100644 index 00000000..c4d0d003 --- /dev/null +++ b/testcases/files/config/apache-auth/digest/.htaccess @@ -0,0 +1,6 @@ +AuthType Digest +AuthName "digest private area" +AuthDigestDomain /digest/ +AuthBasicProvider file +AuthUserFile /var/www/html/digest/.htpasswd +Require valid-user diff --git a/testcases/files/config/apache-auth/digest/.htpasswd b/testcases/files/config/apache-auth/digest/.htpasswd new file mode 100644 index 00000000..cc649515 --- /dev/null +++ b/testcases/files/config/apache-auth/digest/.htpasswd @@ -0,0 +1 @@ +username:digest private area:fad48d3a7c63f61b5b3567a4105bbb04 diff --git a/testcases/files/config/apache-auth/digest_anon/.htaccess b/testcases/files/config/apache-auth/digest_anon/.htaccess new file mode 100644 index 00000000..c8e8648e --- /dev/null +++ b/testcases/files/config/apache-auth/digest_anon/.htaccess @@ -0,0 +1,9 @@ +AuthType Digest +AuthName "digest anon" +AuthDigestDomain /digest_anon/ +AuthBasicProvider file anon +AuthUserFile /var/www/html/digest_anon/.htpasswd +Anonymous_NoUserID off +Anonymous anonymous +Anonymous_LogEmail on +Require valid-user diff --git a/testcases/files/config/apache-auth/digest_anon/.htpasswd b/testcases/files/config/apache-auth/digest_anon/.htpasswd new file mode 100644 index 00000000..47a6af58 --- /dev/null +++ b/testcases/files/config/apache-auth/digest_anon/.htpasswd @@ -0,0 +1,3 @@ +username:digest anon:25e4077a9344ceb1a88f2a62c9fb60d8 +05bbb04 +anonymous:digest anon:faa4e5870970cf935bb9674776e6b26a diff --git a/testcases/files/config/apache-auth/digest_wrongrelm/.htaccess b/testcases/files/config/apache-auth/digest_wrongrelm/.htaccess new file mode 100644 index 00000000..5e7946d2 --- /dev/null +++ b/testcases/files/config/apache-auth/digest_wrongrelm/.htaccess @@ -0,0 +1,6 @@ +AuthType Digest +AuthName "digest private area" +AuthDigestDomain /digest_wrongrelm/ +AuthBasicProvider file +AuthUserFile /var/www/html/digest_wrongrelm/.htpasswd +Require valid-user diff --git a/testcases/files/config/apache-auth/digest_wrongrelm/.htpasswd b/testcases/files/config/apache-auth/digest_wrongrelm/.htpasswd new file mode 100644 index 00000000..019e005d --- /dev/null +++ b/testcases/files/config/apache-auth/digest_wrongrelm/.htpasswd @@ -0,0 +1,2 @@ +username:wrongrelm:99cd340e1283c6d0ab34734bd47bdc30 +4105bbb04 diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index b84631f7..3470f589 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -33,3 +33,10 @@ # failJSON: { "time": "2013-07-17T22:54:32", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:54:32 2013] [error] [client 127.0.0.1] Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match. +# wget --http-user='username' --http-password='wrongpassword' http://localhost/digest/cant_get_me.html -O /dev/null +# failJSON: { "time": "2013-07-17T23:50:37", "match": true , "host": "127.0.0.1" } +[Wed Jul 17 23:50:37 2013] [error] [client 127.0.0.1] Digest: user username: password mismatch: /digest/cant_get_me.html + +# wget --http-user='username' --http-password='password' http://localhost/digest_wrongrelm/cant_get_me.html -O /dev/null +# failJSON: { "time": "2013-07-18T00:08:39", "match": true , "host": "127.0.0.1" } +[Thu Jul 18 00:08:39 2013] [error] [client 127.0.0.1] Digest: user `username' in realm `digest private area' not found: /digest_wrongrelm/cant_get_me.html From 03ec7c211b32b07053246689cf3420049cd88121 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Jul 2013 00:37:33 +1000 Subject: [PATCH 13/60] ENH: could not find a way to trigger filter ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ --- config/filter.d/apache-auth.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 083cfbc8..fa453309 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -33,7 +33,6 @@ failregex = ^%(_apache_error_client)s client denied by server configuration: (ur ^%(_apache_error_client)s user .* not found: \S*\s*$ ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: .*$ - ^%(_apache_error_client)s authorization failure \(no authenticated user\): \S*\s*$ ^%(_apache_error_client)s (Digest: )?user .*: password mismatch: \S*\s*$ ^%(_apache_error_client)s (Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ From fa85be2eea526e6f119723a8e696283c8934ab63 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Jul 2013 08:37:05 +1000 Subject: [PATCH 14/60] DOC/TST: fix configuration path for apache-auth test cases --- testcases/files/logs/apache-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 3470f589..e007b918 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -6,7 +6,7 @@ [Thu Jul 11 01:21:41 2013] [error] [client 194.228.20.113] user not found: / [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: / -# The failures below use the configuration described in testcases/conf/apache-auth +# The failures below use the configuration described in testcases/files/config/apache-auth # # wget http://localhost/noentry/cant_get_me.html -O /dev/null From 5cfe1081865cb9cd413cd252c20e4ae228163169 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 20 Jul 2013 22:21:08 +1000 Subject: [PATCH 15/60] ENH: filter enhancements (with test cases) for apache-auth (httpd-2.4.4) --- config/filter.d/apache-auth.conf | 16 ++++++++-------- config/filter.d/apache-common.conf | 2 +- testcases/files/logs/apache-auth | 26 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index fa453309..4a275a4b 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -28,15 +28,15 @@ before = apache-common.conf # # See also: http://wiki.apache.org/httpd/ListOfErrors # -failregex = ^%(_apache_error_client)s client denied by server configuration: (uri )?\S*\s*$ - ^%(_apache_error_client)s user .* authentication failure for "\S*": Password Mismatch$ - ^%(_apache_error_client)s user .* not found: \S*\s*$ - ^%(_apache_error_client)s client used wrong authentication scheme: \S*\s*$ - ^%(_apache_error_client)s Authorization of user \S+ to access \S* failed, reason: .*$ - ^%(_apache_error_client)s (Digest: )?user .*: password mismatch: \S*\s*$ - ^%(_apache_error_client)s (Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ +failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*\s*$ + ^%(_apache_error_client)s (AH01617: )?user .* authentication failure for "\S*": Password Mismatch$ + ^%(_apache_error_client)s (AH01618: )?user .* not found: \S*\s*$ + ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*\s*$ + ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ + ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*: password mismatch: \S*\s*$ + ^%(_apache_error_client)s (AH0179[01]: )?(Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ - ^%(_apache_error_client)s user .* authorization failure for "\S*": \s*$ + ^%(_apache_error_client)s (AH01631: )?user .* authorization failure for "\S*": \s*$ ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ ^%(_apache_error_client)s invalid nonce .* received - user attempted time travel\s*$ ^%(_apache_error_client)s user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index c3829e2f..ee9c84a0 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -14,4 +14,4 @@ after = apache-common.local [DEFAULT] # Common prefix for [error] apache messages which also would include -_apache_error_client = \[[^]]+\] \[error\] \[client \] +_apache_error_client = \[[^]]+\] \[(\w+:)?error\] (\[pid \d+:tid \d+\] )?\[client (:\d+)?\] diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index e007b918..e6d1aaad 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -13,30 +13,56 @@ # failJSON: { "time": "2013-07-17T23:20:45", "match": true , "host": "127.0.0.1" } [Wed Jul 17 23:20:45 2013] [error] [client 127.0.0.1] client denied by server configuration: /var/www/html/noentry/cant_get_me.html +# failJSON: { "time": "2013-07-20T21:34:49", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:34:49.453232 2013] [access_compat:error] [pid 17512:tid 140123104306944] [client 127.0.0.1:51380] AH01797: client denied by server configuration: /var/www/html/noentry/cant_get_me.html + # wget --http-user='' --http-password='' http://localhost/basic/file/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-17T23:14:37", "match": true , "host": "127.0.0.1" } [Wed Jul 17 23:14:37 2013] [error] [client 127.0.0.1] user not found: /basic/anon/cant_get_me.html +# failJSON: { "time": "2013-07-20T21:37:32", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:37:32.266605 2013] [auth_basic:error] [pid 17512:tid 140123079128832] [client 127.0.0.1:51386] AH01618: user not found: /basic/file/cant_get_me.html + # wget --http-user=username --http-password=wrongpass http://localhost/basic/file -O /dev/null # failJSON: { "time": "2013-07-17T22:18:52", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:18:52 2013] [error] [client 127.0.0.1] user username: authentication failure for "/basic/file": Password Mismatch +# failJSON: { "time": "2013-07-20T21:39:11", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:39:11.978080 2013] [auth_basic:error] [pid 17512:tid 140123053950720] [client 127.0.0.1:51390] AH01617: user username: authentication failure for "/basic/file": Password Mismatch + # wget --http-user=wrongusername --http-password=wrongpass http://localhost/basic/file -O /dev/null # failJSON: { "time": "2013-07-17T22:32:48", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:32:48 2013] [error] [client 127.0.0.1] user wrongusername not found: /basic/file +# failJSON: { "time": "2013-07-20T21:40:33", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:40:33.803528 2013] [auth_basic:error] [pid 17540:tid 140123095914240] [client 127.0.0.1:51395] AH01618: user wrongusername not found: /basic/file + # wget --header='Authorization: Digest username="Mufasa",realm="testrealm@host.com",nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",uri="/dir/index.html",qop=auth,nc=00000001,cnonce="0a4f113b",response="6629fae49393a05397450978507c4ef1",opaque="5ccc069c403ebaf9f0171e9517f40e41"' http://localhost/basic/file -O /dev/null # failJSON: { "time": "2013-07-17T22:39:55", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:39:55 2013] [error] [client 127.0.0.1] client used wrong authentication scheme: /basic/file +# failJSON: { "time": "2013-07-20T21:41:52", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:41:52.523931 2013] [auth_basic:error] [pid 17512:tid 140122964092672] [client 127.0.0.1:51396] AH01614: client used wrong authentication scheme: /basic/file + # wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-17T22:54:32", "match": true , "host": "127.0.0.1" } [Wed Jul 17 22:54:32 2013] [error] [client 127.0.0.1] Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match. +# failJSON: { "time": "2013-07-20T22:11:43", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 22:11:43.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client 127.0.0.1:51548] AH01637: Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match +# failJSON: { "time": "2013-07-20T21:42:44", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:42:44.304159 2013] [authz_core:error] [pid 17484:tid 140123095914240] [client 127.0.0.1:51397] AH01631: user username: authorization failure for "/basic/authz_owner/cant_get_me.html": + # wget --http-user='username' --http-password='wrongpassword' http://localhost/digest/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-17T23:50:37", "match": true , "host": "127.0.0.1" } [Wed Jul 17 23:50:37 2013] [error] [client 127.0.0.1] Digest: user username: password mismatch: /digest/cant_get_me.html +# failJSON: { "time": "2013-07-20T21:44:06", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:44:06.867985 2013] [auth_digest:error] [pid 17540:tid 140123070736128] [client 127.0.0.1:51406] AH01792: user username: password mismatch: /digest/cant_get_me.html + # wget --http-user='username' --http-password='password' http://localhost/digest_wrongrelm/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-18T00:08:39", "match": true , "host": "127.0.0.1" } [Thu Jul 18 00:08:39 2013] [error] [client 127.0.0.1] Digest: user `username' in realm `digest private area' not found: /digest_wrongrelm/cant_get_me.html + +# failJSON: { "time": "2013-07-20T21:45:28", "match": true , "host": "127.0.0.1" } +[Sat Jul 20 21:45:28.890523 2013] [auth_digest:error] [pid 17540:tid 140122972485376] [client 127.0.0.1:51408] AH01790: user `username' in realm `digest private area' not found: /digest_wrongrelm/cant_get_me.html From eb2f0c927257120dfc32d2450fd63f1962f38821 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:31:21 +0100 Subject: [PATCH 16/60] ENH: Improve postfix regex and add more samples --- config/filter.d/postfix.conf | 13 +++++++++++-- testcases/files/logs/postfix | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f92c3619..b70f0377 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = postfix/smtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,8 +22,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = reject: RCPT from (.*)\[\]: 554 - reject: RCPT from (.*)\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index e4d07b58..172bfca4 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -4,3 +4,7 @@ Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= # failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" } Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 : Relay access denied; from= to= proto=SMTP helo=<198.51.100.17> +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.65" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 : Helo command rejected: match bad.domain; from= to= proto=SMTP helo= +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> From 83a80a29ea51d1cb2dc1ce45a33b1d44a8619c9c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:34:00 +0100 Subject: [PATCH 17/60] ENH: Improve couriersmtp and add sample logs --- config/filter.d/couriersmtp.conf | 11 ++++++++++- testcases/files/logs/couriersmtp | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testcases/files/logs/couriersmtp diff --git a/config/filter.d/couriersmtp.conf b/config/filter.d/couriersmtp.conf index 6c0cf5ff..65ffa5d7 100644 --- a/config/filter.d/couriersmtp.conf +++ b/config/filter.d/couriersmtp.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = courieresmtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,7 +22,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = error,relay=,.*550 User unknown +failregex = ^%(__prefix_line)serror,relay=,.*: 550 User unknown\.$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/couriersmtp b/testcases/files/logs/couriersmtp new file mode 100644 index 00000000..212df3b4 --- /dev/null +++ b/testcases/files/logs/couriersmtp @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-10T03:47:57", "match": true , "host": "1.2.3.4" } +Apr 10 03:47:57 web courieresmtpd: error,relay=::ffff:1.2.3.4,ident=tmf,from=,to=: 550 User unknown. +# failJSON: { "time": "2005-07-06T03:42:28", "match": true , "host": "1.2.3.4" } +Jul 6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=: 550 User unknown. +# failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } +Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. +# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. From bd175f026737d66e7110868fb50b3760ff75e087 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:38:29 +0100 Subject: [PATCH 18/60] ENH: Improve cyrus-imap regex and add sample log file --- config/filter.d/cyrus-imap.conf | 16 ++++++++++++---- testcases/files/logs/cyrus-imap | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 testcases/files/logs/cyrus-imap diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index 758f75de..ae7bf6bd 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:cyrus/)?(?:imapd?|pop3d?) + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,10 +22,9 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = : badlogin: .*\[\] plaintext .*SASL\(-13\): authentication failure: checkpass failed$ - : badlogin: .*\[\] LOGIN \[SASL\(-13\): authentication failure: checkpass failed\]$ - : badlogin: .*\[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ - : badlogin: .*\[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ \[\] (?:plaintext|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ + ^%(__prefix_line)sbadlogin: \S+ \[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ + ^%(__prefix_line)sbadlogin: \S+ \[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/cyrus-imap b/testcases/files/logs/cyrus-imap new file mode 100644 index 00000000..5886938e --- /dev/null +++ b/testcases/files/logs/cyrus-imap @@ -0,0 +1,10 @@ +# failJSON: { "time": "2005-01-04T21:51:05", "match": true , "host": "127.0.0.1" } +Jan 4 21:51:05 hostname cyrus/imap[5355]: badlogin: localhost.localdomain [127.0.0.1] plaintext cyrus@localdomain SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "198.51.100.23" } +Feb 20 17:23:32 domain cyrus/pop3[18635]: badlogin: localhost [198.51.100.23] plaintext administrator SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "1.2.3.4" } +Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail0001 SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-06-08T18:11:13", "match": true , "host": "198.51.100.45" } +Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated] +# failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" } +Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response] From 57a6c11260c835b221b88edf8821948128953bed Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:53:18 +0100 Subject: [PATCH 19/60] ENH: Improve courierlogin regex and add sample logs --- config/filter.d/courierlogin.conf | 11 ++++++++++- testcases/files/logs/courierlogin | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testcases/files/logs/courierlogin diff --git a/config/filter.d/courierlogin.conf b/config/filter.d/courierlogin.conf index 20731e5d..f096325e 100644 --- a/config/filter.d/courierlogin.conf +++ b/config/filter.d/courierlogin.conf @@ -5,8 +5,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -14,7 +23,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = LOGIN FAILED, .*, ip=\[\]$ +failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/courierlogin b/testcases/files/logs/courierlogin new file mode 100644 index 00000000..e3d0d8c3 --- /dev/null +++ b/testcases/files/logs/courierlogin @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-23T21:59:01", "match": true , "host": "1.2.3.4" } +Apr 23 21:59:01 dns2 imapd: LOGIN FAILED, user=sales@example.com, ip=[::ffff:1.2.3.4] +# failJSON: { "time": "2005-04-23T21:59:38", "match": true , "host": "198.51.100.76" } +Apr 23 21:59:38 dns2 pop3d: LOGIN FAILED, user=info@example.com, ip=[::ffff:198.51.100.76] +# failJSON: { "time": "2004-11-13T08:11:53", "match": true , "host": "198.51.100.33" } +Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff:198.51.100.33] +# failJSON: { "time": "2005-04-17T19:17:11", "match": true , "host": "1.2.3.4" } +Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4] From ab671b0b1a2c6beec54f8cb585716513210db62e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 16:34:24 +0100 Subject: [PATCH 20/60] ENH: Improve wuftpd failregex, drop duplicate pam regex and add sample For wu-ftpd configured to use pam, the pam filter used be used, as regex is more robust. --- config/filter.d/wuftpd.conf | 11 +++++++++-- testcases/files/logs/wuftpd | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 3351d258..5c1cbb58 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -4,14 +4,21 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] +_daemon = wu-ftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = wu-ftpd(?:\[\d+\])?:\s+\(pam_unix\)\s+authentication failure.* rhost=$ - wu-ftpd(?:\[\d+\])?: *failed login from .*\[\] *$ +failregex = ^%(__prefix_line)sfailed login from \S+ \[\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/wuftpd b/testcases/files/logs/wuftpd index 22ac0303..bbb816cc 100644 --- a/testcases/files/logs/wuftpd +++ b/testcases/files/logs/wuftpd @@ -1,3 +1,5 @@ # This login line is from syslog # failJSON: { "time": "2004-10-06T09:59:26", "match": true , "host": "202.108.145.173" } Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] +# failJSON: { "time": "2004-10-11T16:45:07", "match": true , "host": "198.51.100.71" } +Oct 11 16:45:07 ubuntu wu-ftpd[2360]: failed login from example.com [198.51.100.71] From d791ba12bab3cbf7c1d804cff91b6dc30a254bcc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 16:54:28 +0100 Subject: [PATCH 21/60] TST: Add sample log for dropbear filter --- testcases/files/logs/dropbear | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 testcases/files/logs/dropbear diff --git a/testcases/files/logs/dropbear b/testcases/files/logs/dropbear new file mode 100644 index 00000000..5a2e197a --- /dev/null +++ b/testcases/files/logs/dropbear @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 From 534be189dcd43f93182ee14f32ce78e67e5c5084 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 17:26:09 +0100 Subject: [PATCH 22/60] ENH: Improve sieve regex and add sample line --- config/filter.d/sieve.conf | 11 ++++++++++- testcases/files/logs/sieve | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sieve.conf b/config/filter.d/sieve.conf index 866b4228..83608fa6 100644 --- a/config/filter.d/sieve.conf +++ b/config/filter.d/sieve.conf @@ -4,15 +4,24 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_deamon = (?:cyrus/)?(?:tim)?sieved? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can # be used for standard IP/hostname matching. # Values: TEXT # -failregex = : badlogin: .*\[\] (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failure$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:LOGIN|PLAIN|OTP|(?:CRAM|DIGEST)-MD5) authentication failure$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/sieve b/testcases/files/logs/sieve index 5cc19673..770d17ad 100644 --- a/testcases/files/logs/sieve +++ b/testcases/files/logs/sieve @@ -1,2 +1,6 @@ # failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" } Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2005-07-18T17:21:58", "match": true , "host": "1.2.3.4" } +Jul 18 17:21:58 ophelia cyrus/timsieved[12305]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2004-09-25T22:07:38", "match": true , "host": "1.2.3.4" } +Sep 25 22:07:38 web9 timsieved[21040]: badlogin: web4[1.2.3.4] OTP authentication failure From a11f91b83506289d03c4df2bf3c5ea3cad27f665 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 17:28:28 +0100 Subject: [PATCH 23/60] ENH: Improve cyrus-imap regex and add extra sample line --- config/filter.d/cyrus-imap.conf | 5 ++--- testcases/files/logs/cyrus-imap | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index ae7bf6bd..b63fad8f 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -22,9 +22,8 @@ _daemon = (?:cyrus/)?(?:imapd?|pop3d?) # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ \[\] (?:plaintext|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ - ^%(__prefix_line)sbadlogin: \S+ \[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ - ^%(__prefix_line)sbadlogin: \S+ \[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:plaintext|PLAIN|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ + ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:(?:CRAM|DIGEST)-MD5|NTLM|OTP) \[SASL\(-13\): authentication failure: .*\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/cyrus-imap b/testcases/files/logs/cyrus-imap index 5886938e..9bf271f6 100644 --- a/testcases/files/logs/cyrus-imap +++ b/testcases/files/logs/cyrus-imap @@ -8,3 +8,6 @@ Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated] # failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" } Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response] +# failJSON: { "time": "2004-12-30T16:03:27", "match": true , "host": "1.2.3.4" } +Dec 30 16:03:27 somehost imapd[2517]: badlogin: local-somehost[1.2.3.4] OTP [SASL(-13): authentication failure: External SSF not good enough] + From e7b7815de33446a8705dac69326bb167fcd302ec Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:22:44 +0100 Subject: [PATCH 24/60] TST: Add additional sshd sample logs --- testcases/files/logs/sshd | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index ff97c5a5..75854774 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -77,3 +77,11 @@ Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error #11 https://github.com/fail2ban/fail2ban/issues/267 There might be no colon after [daemon] # failJSON: { "time": "2005-06-25T23:53:34", "match": true , "host": "1.2.3.4" } Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers +# failJSON: { "time": "2004-12-12T20:04:39", "match": true , "host": "10.215.4.227" } +Dec 12 20:04:39 aragorn sshd[1328]: error: PAM: User not known to the underlying authentication module for illegal user kernelitshell from 10.215.4.227 +# failJSON: { "time": "2005-03-26T04:56:27", "match": true , "host": "example.com" } +Mar 26 04:56:27 angel sshd[9739]: User allena from example.com not allowed because not in any group +# failJSON: { "time": "2005-02-07T16:01:07", "match": true , "host": "192.51.100.54" } +Feb 7 16:01:07 linux-m899 sshd[5106]: User root from 192.51.100.54 not allowed because a group is listed in DenyGroups +# failJSON: { "time": "2005-01-05T11:15:05", "match": true , "host": "10.0.0.40" } +Jan 5 11:15:05 NAS sshd[1966]: User root from 10.0.0.40 not allowed because none of user's groups are listed in AllowGroups From b5ffbced37b3000aae229f0bcac88029646be4b2 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:31:32 +0100 Subject: [PATCH 25/60] TST: Sample test cases now handle ignoreregex and add recidive samples --- testcases/files/logs/recidive | 6 ++++++ testcases/samplestestcase.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 testcases/files/logs/recidive diff --git a/testcases/files/logs/recidive b/testcases/files/logs/recidive new file mode 100644 index 00000000..cf6df933 --- /dev/null +++ b/testcases/files/logs/recidive @@ -0,0 +1,6 @@ +# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [sendmail] Ban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 16:07:31,183 fail2ban.actions: WARNING [sendmail] Unban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [recidive] Ban 1.2.3.4 diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index a52873b9..dd478fbf 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -65,6 +65,8 @@ def testSampleRegexsFactory(name): for opt in filterConf.convert(): if opt[2] == "addfailregex": self.filter.addFailRegex(opt[3]) + elif opt[2] == "addignoreregex": + self.filter.addIgnoreRegex(opt[3]) if not self.filter.getFailRegex(): # No fail regexs set: likely just common file for includes. From 4033857f63cdb5702bf15e436a7360880d8a1676 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:44:09 +0100 Subject: [PATCH 26/60] ENH: Improve xinetd-fail regex and add sample logs --- config/filter.d/xinetd-fail.conf | 13 +++++++++++-- testcases/files/logs/xinetd-fail | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 testcases/files/logs/xinetd-fail diff --git a/config/filter.d/xinetd-fail.conf b/config/filter.d/xinetd-fail.conf index 4ff5bfde..253ce15d 100644 --- a/config/filter.d/xinetd-fail.conf +++ b/config/filter.d/xinetd-fail.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = xinetd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -19,8 +28,8 @@ # load => xinetd: max_load (temporary problem) # -failregex = xinetd(?:\[\d{1,5}\])?: FAIL: \S+ address from=$ - xinetd(?:\[\d{1,5}\])?: FAIL: \S+ libwrap from=$ +failregex = ^%(__prefix_line)sFAIL: \S+ address from=$ + ^%(__prefix_line)sFAIL: \S+ libwrap from=$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/xinetd-fail b/testcases/files/logs/xinetd-fail new file mode 100644 index 00000000..8545a067 --- /dev/null +++ b/testcases/files/logs/xinetd-fail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-05-15T17:38:49", "match": true , "host": "198.51.100.169" } +May 15 17:38:49 boo xinetd[16256]: FAIL: telnet address from=198.51.100.169 +# failJSON: { "time": "2005-08-03T14:38:49", "match": true , "host": "198.51.100.223" } +Aug 3 14:38:49 backup xinetd[31234]: FAIL: amanda libwrap from=198.51.100.223 From 8b9bafda7988d007b8e02dc21e17ea8fd6b07c30 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 16:31:11 +0100 Subject: [PATCH 27/60] ENH: Change lighttpd-fastcgi to suhosin, and improve regex and samples suhosin is hardened php implmentation, which will log the alerts (as seen in samples) to stderr, which is picked up by fastcgi webserver (e.g. lighttpd, apache, nginx) --- MANIFEST | 2 +- .../{lighttpd-fastcgi.conf => suhosin.conf} | 3 ++- config/jail.conf | 16 +++------------- testcases/files/logs/suhosin | 4 ++++ 4 files changed, 10 insertions(+), 15 deletions(-) rename config/filter.d/{lighttpd-fastcgi.conf => suhosin.conf} (67%) create mode 100644 testcases/files/logs/suhosin diff --git a/MANIFEST b/MANIFEST index 5491c7d5..f637dca0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -101,7 +101,7 @@ config/filter.d/couriersmtp.conf config/filter.d/cyrus-imap.conf config/filter.d/exim.conf config/filter.d/gssftpd.conf -config/filter.d/lighttpd-fastcgi.conf +config/filter.d/suhosin.conf config/filter.d/named-refused.conf config/filter.d/postfix.conf config/filter.d/proftpd.conf diff --git a/config/filter.d/lighttpd-fastcgi.conf b/config/filter.d/suhosin.conf similarity index 67% rename from config/filter.d/lighttpd-fastcgi.conf rename to config/filter.d/suhosin.conf index 1c6e3fce..c79c157f 100644 --- a/config/filter.d/lighttpd-fastcgi.conf +++ b/config/filter.d/suhosin.conf @@ -9,7 +9,8 @@ # Notes.: regex to match ALERTS as notified by lighttpd's FastCGI Module # Values: TEXT # -failregex = .*ALERT\ -\ .*attacker\ \'\' +# https://github.com/stefanesser/suhosin/blob/1fba865ab73cc98a3109f88d85eb82c1bfc29b37/log.c#L161 +failregex = ALERT - .* \(attacker '', file '.*'(?:, line \d+)?\)$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/jail.conf b/config/jail.conf index e3b92038..640a5874 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -256,25 +256,15 @@ filter = php-url-fopen logpath = /var/www/*/logs/access_log maxretry = 1 -# A simple PHP-fastcgi jail which works with lighttpd. -# If you run a lighttpd server, then you probably will -# find these kinds of messages in your error_log: -# ALERT – tried to register forbidden variable ‘GLOBALS’ -# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') -# This jail would block the IP 1.2.3.4. - -[lighttpd-fastcgi] +[suhosin] enabled = false -filter = lighttpd-fastcgi -action = iptables-multiport[name=lighttpd-fastcgi, port="http,https"] +filter = suhosin +action = iptables-multiport[name=suhosin, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 -# Same as above for mod_auth -# It catches wrong authentications - [lighttpd-auth] enabled = false diff --git a/testcases/files/logs/suhosin b/testcases/files/logs/suhosin new file mode 100644 index 00000000..90ed7bf1 --- /dev/null +++ b/testcases/files/logs/suhosin @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-03-11T22:52:12", "match": true , "host": "198.51.100.167" } +Mar 11 22:52:12 lighttpd[53690]: (mod_fastcgi.c.2676) FastCGI-stderr: ALERT - configured request variable name length limit exceeded - dropped variable 'upqchi07vFfAFuBjnIKGIwiLrHo3Vt68T3yqvhQu2TqetQ78roy7Q6bpTfDUtYFR593/MA' (attacker '198.51.100.167', file '/usr/local/captiveportal/index.php') +# failJSON: { "time": "2005-02-26T22:52:29", "match": true , "host": "198.51.100.77" } +Feb 26 22:52:29 host suhosin[9636]: ALERT - script tried to increase memory_limit to 268435456 bytes which is above the allowed value (attacker '198.51.100.77', file '/var/www/wordpress/wp-admin/includes/image.php', line 161) From e59a4960a3bea0e90fcb01ffcc03092ebf1c10f3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 16:48:12 +0100 Subject: [PATCH 28/60] TST: Add additional sample log line for apache-noscript --- testcases/files/logs/apache-noscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/files/logs/apache-noscript b/testcases/files/logs/apache-noscript index e08b3468..172bb510 100644 --- a/testcases/files/logs/apache-noscript +++ b/testcases/files/logs/apache-noscript @@ -1,2 +1,4 @@ # failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" } [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat +# failJSON: { "time": "2005-07-22T06:48:30", "match": true , "host": "198.51.100.86" } +[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php From cf1e5bdbc223384340b2457b9eba08524f924ba8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 22:03:49 +0100 Subject: [PATCH 29/60] ENH: Tweak proftpd regex and add sample logs Needed to add optional ":" post __pid_re, and for consistency, decided to make use of __prefix_line instead which includes this. --- config/filter.d/proftpd.conf | 10 ++++++---- testcases/files/logs/proftpd | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 13080fcc..d17658ce 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -13,6 +13,8 @@ before = common.conf [Definition] +_deamon = proftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -20,10 +22,10 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ +failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index 404ebfaf..9687d992 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -10,6 +10,7 @@ Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting. Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. # failJSON: { "time": "2005-06-14T00:09:59", "match": true , "host": "59.167.242.100" } Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 - - - +# failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } +May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 From 37f240bef00febcbe2711014ba1c4d53dab5070c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 22:13:37 +0100 Subject: [PATCH 30/60] TST: Add sample log for php-url-fopen filter --- testcases/files/logs/php-url-fopen | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/php-url-fopen diff --git a/testcases/files/logs/php-url-fopen b/testcases/files/logs/php-url-fopen new file mode 100644 index 00000000..f119a928 --- /dev/null +++ b/testcases/files/logs/php-url-fopen @@ -0,0 +1,2 @@ +# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" } +66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" From b5639a8672b3ace9c3e761d05fc76b50c29d3b17 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 11:55:09 +0100 Subject: [PATCH 31/60] ENH: Simplify cyrus-imap filter fail regex --- config/filter.d/cyrus-imap.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index b63fad8f..0ace92c1 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -22,8 +22,7 @@ _daemon = (?:cyrus/)?(?:imapd?|pop3d?) # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:plaintext|PLAIN|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ - ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:(?:CRAM|DIGEST)-MD5|NTLM|OTP) \[SASL\(-13\): authentication failure: .*\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ .*?\[?SASL\(-13\): authentication failure: .*\]?$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 2f4aaa9fb9a26b89149a5a7e831f9a7bd765d80f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 12:01:09 +0100 Subject: [PATCH 32/60] ENH: Simplify sieve filter failregex --- config/filter.d/sieve.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sieve.conf b/config/filter.d/sieve.conf index 83608fa6..b2af6774 100644 --- a/config/filter.d/sieve.conf +++ b/config/filter.d/sieve.conf @@ -21,7 +21,7 @@ _deamon = (?:cyrus/)?(?:tim)?sieved? # be used for standard IP/hostname matching. # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:LOGIN|PLAIN|OTP|(?:CRAM|DIGEST)-MD5) authentication failure$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ authentication failure$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From f7d8e6873885ec37843a346171a544dc6a4e4600 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 12:32:29 +0100 Subject: [PATCH 33/60] TST: Add apache-badbots sample log --- testcases/files/logs/apache-badbots | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/apache-badbots diff --git a/testcases/files/logs/apache-badbots b/testcases/files/logs/apache-badbots new file mode 100644 index 00000000..35669252 --- /dev/null +++ b/testcases/files/logs/apache-badbots @@ -0,0 +1,2 @@ +# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } +1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" From 5437f5fe908d3a95926d058e0fb4da9b4ab829d6 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:02:53 +0100 Subject: [PATCH 34/60] TST: Add gssftpd sample log --- testcases/files/logs/gssftpd | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/gssftpd diff --git a/testcases/files/logs/gssftpd b/testcases/files/logs/gssftpd new file mode 100644 index 00000000..58fda7dc --- /dev/null +++ b/testcases/files/logs/gssftpd @@ -0,0 +1,2 @@ +# failJSON: { "time": "2005-01-22T18:09:46", "match": true , "host": "198.51.100.23" } +Jan 22 18:09:46 host ftpd[132]: repeated login failures from 198.51.100.23 (example.com) From 1c7d28d1ea8e79dd8bd11f7899a2a4d4be1f4830 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:03:14 +0100 Subject: [PATCH 35/60] TST: Add qmail sample log --- testcases/files/logs/qmail | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 testcases/files/logs/qmail diff --git a/testcases/files/logs/qmail b/testcases/files/logs/qmail new file mode 100644 index 00000000..63cad4cc --- /dev/null +++ b/testcases/files/logs/qmail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2004-09-06T07:33:33", "match": true , "host": "198.51.100.77" } +Sep 6 07:33:33 sd6 qmail: 1157520813.485077 rblsmtpd: 198.51.100.77 pid 19597 sbl-xbl.spamhaus.org: 451 http://www.spamhaus.org/query/bl?ip=198.51.100.77 +# failJSON: { "time": "2004-09-06T07:18:29", "match": true , "host": "198.51.100.54" } +Sep 6 07:18:29 sd6 qmail: 1157519909.633171 qmail-smtpd: 421 badiprbl: ip 198.51.100.54 rbl: example.com From bf021ebd97719c9a1b7436366f4379f3d86d71ed Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:05:17 +0100 Subject: [PATCH 36/60] TST: Mandate that all filters and each regex has sample log entry --- testcases/samplestestcase.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index dd478fbf..54afc864 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -72,14 +72,9 @@ def testSampleRegexsFactory(name): # No fail regexs set: likely just common file for includes. return - # TODO: Remove exception handling once sample logs obtained for all - try: - self.assertTrue( - os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), - "No sample log file available for '%s' filter" % name) - except AssertionError: - print "I: No sample log file available for '%s' filter" % name - return + self.assertTrue( + os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), + "No sample log file available for '%s' filter" % name) logFile = fileinput.FileInput( os.path.join(TEST_FILES_DIR, "logs", name)) @@ -124,14 +119,10 @@ def testSampleRegexsFactory(name): # TODO: Remove exception handling once all regexs have samples for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): - try: - self.assertTrue( - failRegexIndex in regexsUsed, - "Regex for filter '%s' has no samples: %i: %r" % - (name, failRegexIndex, failRegex)) - except AssertionError: - print "I: Regex for filter '%s' has no samples: %i: %r" % ( - name, failRegexIndex, failRegex) + self.assertTrue( + failRegexIndex in regexsUsed, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) return testFilter From 11100a2fc0c65677f3e5d7b065c012aa37cbd547 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:20:05 +0100 Subject: [PATCH 37/60] DOC: Changelog additions for regex changes --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index a6a6ab23..237ef672 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,10 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Steven Hiscocks * filter.d/dovecot - Addition of session, time values and possible blank user + * filter.d/{courier{login,smtp},proftpd,sieve,wuftpd,xinetd} - General + regex impovements + * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which + will require changing in jail.{conf,local} if using this filter. ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From 0fb04cb2f043061adf4dd521794253ae0c286d5c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 28 Jul 2013 22:00:55 +1000 Subject: [PATCH 38/60] ENH: filter enhancements on mod-digest (with test cases) for apache-auth (httpd-2.4.4) --- config/filter.d/apache-auth.conf | 15 ++- testcases/files/config/apache-auth/digest.py | 99 ++++++++++++++++++ .../config/apache-auth/digest_time/.htaccess | 7 ++ .../apache-auth/digest_time/.htaccess.swp | Bin 0 -> 12288 bytes .../config/apache-auth/digest_time/.htpasswd | 1 + testcases/files/logs/apache-auth | 32 ++++++ 6 files changed, 146 insertions(+), 8 deletions(-) create mode 100755 testcases/files/config/apache-auth/digest.py create mode 100644 testcases/files/config/apache-auth/digest_time/.htaccess create mode 100644 testcases/files/config/apache-auth/digest_time/.htaccess.swp create mode 100644 testcases/files/config/apache-auth/digest_time/.htpasswd diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 4a275a4b..d1a2ffcb 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -35,15 +35,14 @@ failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server config ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*: password mismatch: \S*\s*$ ^%(_apache_error_client)s (AH0179[01]: )?(Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ - ^%(_apache_error_client)s user .* authorization failure: \S*\s*$ ^%(_apache_error_client)s (AH01631: )?user .* authorization failure for "\S*": \s*$ - ^%(_apache_error_client)s invalid nonce .* received - (length|hash) is not \S+\s*$ - ^%(_apache_error_client)s invalid nonce .* received - user attempted time travel\s*$ - ^%(_apache_error_client)s user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ - ^%(_apache_error_client)s user .*: one-time-nonce mismatch - sending new nonce\s*$ - ^%(_apache_error_client)s realm mismatch - got `.*' but expected `.+'\s*$ - ^%(_apache_error_client)s unknown algorithm `\S+' received: \S*\s*"$ - ^%(_apache_error_client)s invalid qop `.*' received: \S*\s*"$ + ^%(_apache_error_client)s (AH0177[56]: )?invalid nonce .* received - (length|hash) is not \S+\s*$ + ^%(_apache_error_client)s (AH01788: )?realm mismatch - got `.*' but expected `.+'\s*$ + ^%(_apache_error_client)s (AH01789: )?unknown algorithm `\S+' received: \S*\s*"$ + ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*"$ + ^%(_apache_error_client)s (AH01777: )?invalid nonce .* received - user attempted time travel\s*$ + ^%(_apache_error_client)s (AH01778: )?user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ + ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$ diff --git a/testcases/files/config/apache-auth/digest.py b/testcases/files/config/apache-auth/digest.py new file mode 100755 index 00000000..ed0e18eb --- /dev/null +++ b/testcases/files/config/apache-auth/digest.py @@ -0,0 +1,99 @@ +#!/bin/env python +import requests +import md5 + + +def auth(v): + + ha1 = md5.new(username + ':' + realm + ':' + password).hexdigest() + ha2 = md5.new("GET:" + url).hexdigest() + + #response = md5.new(ha1 + ':' + v['nonce'][1:-1] + ':' + v['nc'] + ':' + v['cnonce'][1:-1] + # + ':' + v['qop'][1:-1] + ':' + ha2).hexdigest() + + nonce = v['nonce'][1:-1] + nc=v.get('nc') or '' + cnonce = v.get('cnonce') or '' + qop = v['qop'][1:-1] + algorithm = v['algorithm'] + response = md5.new(ha1 + ':' + nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2).hexdigest() + + p = requests.Request('GET', host + url).prepare() + #p.headers['Authentication-Info'] = response + p.headers['Authorization'] = """ + Digest username="%s", + algorithm="%s", + realm="%s", + uri="%s", + nonce="%s", + cnonce="", + nc="", + qop=%s, + response="%s" + """ % ( username, algorithm, realm, url, nonce, qop, response ) + + s = requests.Session() + return s.send(p) + +def preauth(): + r = requests.get(host + url) + r.headers['www-authenticate'].split(', ') + return dict([ a.split('=',1) for a in r.headers['www-authenticate'].split(', ') ]) + + +url='/digest/' +host = 'http://localhost:801' + +v = preauth() + +#print v +username="username" +password = "password" + +realm = 'so far away' +r = auth(v) + +realm = v['Digest realm'][1:-1] + +# [Sun Jul 28 21:27:56.549667 2013] [auth_digest:error] [pid 24835:tid 139895297222400] [client 127.0.0.1:57052] AH01788: realm mismatch - got `so far away' but expected `digest private area' + + +algorithm = v['algorithm'] +v['algorithm'] = 'super funky chicken' +r = auth(v) + +# [Sun Jul 28 21:41:20 2013] [error] [client 127.0.0.1] Digest: unknown algorithm `super funky chicken' received: /digest/ + +print r.status_code,r.headers, r.text +v['algorithm'] = algorithm + + +r = auth(v) +print r.status_code,r.headers, r.text + +nonce = v['nonce'] +v['nonce']=v['nonce'][5:-5] + +r = auth(v) +print r.status_code,r.headers, r.text + +# [Sun Jul 28 21:05:31.178340 2013] [auth_digest:error] [pid 24224:tid 139895539455744] [client 127.0.0.1:56906] AH01793: invalid qop `auth' received: /digest/qop_none/ + + +v['nonce']=nonce[0:11] + 'ZZZ' + nonce[14:] + +r = auth(v) +print r.status_code,r.headers, r.text + +#[Sun Jul 28 21:18:11.769228 2013] [auth_digest:error] [pid 24752:tid 139895505884928] [client 127.0.0.1:56964] AH01776: invalid nonce b9YAiJDiBAZZZ1b1abe02d20063ea3b16b544ea1b0d981c1bafe received - hash is not d42d824dee7aaf50c3ba0a7c6290bd453e3dd35b + + +url='/digest_time/' +v=preauth() + +import time +time.sleep(1) + +r = auth(v) +print r.status_code,r.headers, r.text + diff --git a/testcases/files/config/apache-auth/digest_time/.htaccess b/testcases/files/config/apache-auth/digest_time/.htaccess new file mode 100644 index 00000000..44036f57 --- /dev/null +++ b/testcases/files/config/apache-auth/digest_time/.htaccess @@ -0,0 +1,7 @@ +AuthType Digest +AuthName "digest private area" +AuthDigestDomain /digest_time/ +AuthBasicProvider file +AuthUserFile /var/www/html/digest_time/.htpasswd +AuthDigestNonceLifetime 1 +Require valid-user diff --git a/testcases/files/config/apache-auth/digest_time/.htaccess.swp b/testcases/files/config/apache-auth/digest_time/.htaccess.swp new file mode 100644 index 0000000000000000000000000000000000000000..1d14e6f29e6247f67d618dc791a6b6791672ed5f GIT binary patch literal 12288 zcmeI&!Aiq07zglgL-8bvdx1F5Tc-V)bjHm$CNpJoiR63cWzxNMo<9ilo;ZS@wNt%*?Z}00I!0Lm;)o#@;Hm zceYza_EvpU)HeEaY=jgDKmY;|fB*y_009UkNo&n})^+`8zwDAw(}K!X4T zAOHafKmY;|fB*y_009X68-WXXmntnq#-R!tsgarvQ#piDSoqx_zZGZjcJhA#EX37xXiUVN*hFQ>cgPOMJ~>_wtg##Rc} PlI!L4<$WT> Date: Sun, 28 Jul 2013 22:01:51 +1000 Subject: [PATCH 39/60] TST: bad include of vim swap files --- .../apache-auth/digest_time/.htaccess.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 testcases/files/config/apache-auth/digest_time/.htaccess.swp diff --git a/testcases/files/config/apache-auth/digest_time/.htaccess.swp b/testcases/files/config/apache-auth/digest_time/.htaccess.swp deleted file mode 100644 index 1d14e6f29e6247f67d618dc791a6b6791672ed5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI&!Aiq07zglgL-8bvdx1F5Tc-V)bjHm$CNpJoiR63cWzxNMo<9ilo;ZS@wNt%*?Z}00I!0Lm;)o#@;Hm zceYza_EvpU)HeEaY=jgDKmY;|fB*y_009UkNo&n})^+`8zwDAw(}K!X4T zAOHafKmY;|fB*y_009X68-WXXmntnq#-R!tsgarvQ#piDSoqx_zZGZjcJhA#EX37xXiUVN*hFQ>cgPOMJ~>_wtg##Rc} PlI!L4<$WT> Date: Mon, 29 Jul 2013 02:29:04 +1000 Subject: [PATCH 40/60] TST: apache-auth filter - nonce timetravel tests + other expression fixes --- config/filter.d/apache-auth.conf | 10 ++-- testcases/files/config/apache-auth/digest.py | 54 +++++++++++++++++++- testcases/files/logs/apache-auth | 11 ++++ 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index d1a2ffcb..0db34523 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -35,11 +35,11 @@ failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server config ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*: password mismatch: \S*\s*$ ^%(_apache_error_client)s (AH0179[01]: )?(Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ - ^%(_apache_error_client)s (AH01631: )?user .* authorization failure for "\S*": \s*$ - ^%(_apache_error_client)s (AH0177[56]: )?invalid nonce .* received - (length|hash) is not \S+\s*$ - ^%(_apache_error_client)s (AH01788: )?realm mismatch - got `.*' but expected `.+'\s*$ - ^%(_apache_error_client)s (AH01789: )?unknown algorithm `\S+' received: \S*\s*"$ - ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*"$ + ^%(_apache_error_client)s (AH01631: )?user .*: authorization failure for "\S*":\s*$ + ^%(_apache_error_client)s (AH0177[56]: )?(Digest: )?invalid nonce .* received - (length|hash) is not \S+\s*$ + ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*' but expected `.+'\s*$ + ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*' received: \S*\s*$ + ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*$ ^%(_apache_error_client)s (AH01777: )?invalid nonce .* received - user attempted time travel\s*$ ^%(_apache_error_client)s (AH01778: )?user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$ diff --git a/testcases/files/config/apache-auth/digest.py b/testcases/files/config/apache-auth/digest.py index ed0e18eb..c12b1b33 100755 --- a/testcases/files/config/apache-auth/digest.py +++ b/testcases/files/config/apache-auth/digest.py @@ -37,18 +37,19 @@ def auth(v): def preauth(): r = requests.get(host + url) + print r r.headers['www-authenticate'].split(', ') return dict([ a.split('=',1) for a in r.headers['www-authenticate'].split(', ') ]) url='/digest/' -host = 'http://localhost:801' +host = 'http://localhost:802' v = preauth() -#print v username="username" password = "password" +print v realm = 'so far away' r = auth(v) @@ -97,3 +98,52 @@ time.sleep(1) r = auth(v) print r.status_code,r.headers, r.text +# Obtained by putting the following code in modules/aaa/mod_auth_digest.c +# in the function initialize_secret +# { +# const char *hex = "0123456789abcdef"; +# char secbuff[SECRET_LEN * 4]; +# char *hash = secbuff; +# int idx; + +# for (idx=0; idx> 4]; +# *hash++ = hex[secret[idx] & 0xF]; +# } +# *hash = '\0'; +# /* remove comment in below for apache-2.4+ */ +# ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, s, /* APLOGNO(11759) */ "secret: %s", secbuff); +# } + + +import sha +import binascii +import base64 +import struct + +apachesecret = binascii.unhexlify('cc969f83b4029e672115f2e8ff7dd21a976728f9') +s = sha.sha(apachesecret) + +v=preauth() + +print v['nonce'] +realm = v['Digest realm'][1:-1] + +(t,) = struct.unpack('l',base64.b64decode(v['nonce'][1:13])) + +# whee, time travel +t = t + 5540 + +timepac = base64.b64encode(struct.pack('l',t)) + +s.update(realm) +s.update(timepac) + +v['nonce'] = v['nonce'][0] + timepac + s.hexdigest() + v['nonce'][-1] + +print v + +r = auth(v) +#[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel +print r.status_code,r.headers, r.text + diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 95397cb5..915fabc1 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -50,6 +50,8 @@ # failJSON: { "time": "2013-07-20T22:11:43", "match": true , "host": "127.0.0.1" } [Sat Jul 20 22:11:43.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client 127.0.0.1:51548] AH01637: Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match + +# wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-20T21:42:44", "match": true , "host": "127.0.0.1" } [Sat Jul 20 21:42:44.304159 2013] [authz_core:error] [pid 17484:tid 140123095914240] [client 127.0.0.1:51397] AH01631: user username: authorization failure for "/basic/authz_owner/cant_get_me.html": @@ -98,3 +100,12 @@ # failJSON: { "time": "2013-07-28T21:42:03", "match": true , "host": "127.0.0.1" } [Sun Jul 28 21:42:03.930190 2013] [auth_digest:error] [pid 24835:tid 139895505884928] [client 127.0.0.1:57115] AH01789: unknown algorithm `super funky chicken' received: /digest/ + +# ./testcases/files/config/apache-auth/digest.py +# failJSON: { "time": "2013-07-29T02:15:26", "match": true , "host": "127.0.0.1" } +[Mon Jul 29 02:15:26 2013] [error] [client 127.0.0.1] Digest: invalid nonce LWEDr5TiBAA=ceddd011628c30e3646f7acda4f1a0ab6b7c5ae6 received - user attempted time travel + + + +# failJSON: { "time": "2013-07-29T02:12:55", "match": true , "host": "127.0.0.1" } +[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel From d8883f4346400c2cef937497b12e87df0a6d2876 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 29 Jul 2013 08:59:25 +1000 Subject: [PATCH 41/60] DOC: Notes about 401 responses and how apache logs this --- config/filter.d/apache-auth.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 0db34523..2b1397cf 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -16,8 +16,9 @@ before = apache-common.conf # from the modules in aaa that return HTTP_UNAUTHORIZED, HTTP_METHOD_NOT_ALLOWED or # HTTP_FORBIDDEN and not AUTH_GENERAL_ERROR or HTTP_INTERNAL_SERVER_ERROR. # -# As an unauthorized response is the first step for a browser to instigate authentication -# so don't set max retry too low. +# An unauthorized response 401 is the first step for a browser to instigate authentication +# however apache doesn't log this as an error. Only subsequent errors are logged in the +# error log. # # Source: # From c0a2e5055909132006371dc60419a4bb3384bfa2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 6 Aug 2013 17:13:09 +1000 Subject: [PATCH 42/60] TST: apache auth - opaque value --- testcases/files/config/apache-auth/digest.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/testcases/files/config/apache-auth/digest.py b/testcases/files/config/apache-auth/digest.py index c12b1b33..bed4067c 100755 --- a/testcases/files/config/apache-auth/digest.py +++ b/testcases/files/config/apache-auth/digest.py @@ -14,6 +14,7 @@ def auth(v): nonce = v['nonce'][1:-1] nc=v.get('nc') or '' cnonce = v.get('cnonce') or '' + opaque = v.get('opaque') or '' qop = v['qop'][1:-1] algorithm = v['algorithm'] response = md5.new(ha1 + ':' + nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2).hexdigest() @@ -31,7 +32,8 @@ def auth(v): qop=%s, response="%s" """ % ( username, algorithm, realm, url, nonce, qop, response ) - +# opaque="%s", + print p.method, p.url, p.headers s = requests.Session() return s.send(p) @@ -43,7 +45,7 @@ def preauth(): url='/digest/' -host = 'http://localhost:802' +host = 'http://localhost:801' v = preauth() @@ -111,7 +113,7 @@ print r.status_code,r.headers, r.text # *hash++ = hex[secret[idx] & 0xF]; # } # *hash = '\0'; -# /* remove comment in below for apache-2.4+ */ +# /* remove comment makings in below for apache-2.4+ */ # ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, s, /* APLOGNO(11759) */ "secret: %s", secbuff); # } @@ -121,7 +123,7 @@ import binascii import base64 import struct -apachesecret = binascii.unhexlify('cc969f83b4029e672115f2e8ff7dd21a976728f9') +apachesecret = binascii.unhexlify('497d8894adafa5ec7c8c981ddf9c8457da7a90ac') s = sha.sha(apachesecret) v=preauth() @@ -147,3 +149,11 @@ r = auth(v) #[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel print r.status_code,r.headers, r.text +url='/digest_onetime/' +v=preauth() + +# Need opaque header handling in auth +r = auth(v) +print r.status_code,r.headers, r.text +r = auth(v) +print r.status_code,r.headers, r.text From 9002de069ebe13a59ee9bada65763120b4290a4f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 9 Aug 2013 18:54:08 +0100 Subject: [PATCH 43/60] ENH: Improve proftpd regex. Taken from @yarikoptic comment: https://github.com/fail2ban/fail2ban/pull/303#discussion_r5687500 --- config/filter.d/proftpd.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index d17658ce..872ab0ca 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -22,8 +22,10 @@ _deamon = proftpd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # + +__suffix_failed_login = (User not authorized|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ From 53e16e07addb827ab11e608cd4204f0513f09ec4 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 9 Aug 2013 19:04:26 +0100 Subject: [PATCH 44/60] ENH: Minor tweak on previous commit proftpd regex changes --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 872ab0ca..d3120216 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -23,7 +23,7 @@ _deamon = proftpd # Values: TEXT # -__suffix_failed_login = (User not authorized|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ From 61d43608ae4d6a142469c91ec4c0611cf117a08d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 18:32:07 +1000 Subject: [PATCH 45/60] ENH: filter.d/postfix - add filter for VRFY. Closes gh-322 --- ChangeLog | 3 +++ config/filter.d/postfix.conf | 1 + testcases/files/logs/postfix | 2 ++ 3 files changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 237ef672..0252e9fd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -63,6 +63,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests regex impovements * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which will require changing in jail.{conf,local} if using this filter. + Zurd + * filter.d/postfix - add filter for VRFY failures. closes gh-322. + ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index b70f0377..da981733 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -24,6 +24,7 @@ _daemon = postfix/smtpd # failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 172bfca4..122ad8e5 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -8,3 +8,5 @@ Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-e Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 : Helo command rejected: match bad.domain; from= to= proto=SMTP helo= # failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" } Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> +# failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" } +Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab From ed42b08789525afb1138b9b0508aad22cdfb4839 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 21:22:17 +1000 Subject: [PATCH 46/60] TST: merge dropbear log samples --- testcases/files/logs/dropbear | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testcases/files/logs/dropbear b/testcases/files/logs/dropbear index 5a2e197a..139292d7 100644 --- a/testcases/files/logs/dropbear +++ b/testcases/files/logs/dropbear @@ -2,3 +2,8 @@ Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 # failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 + From ea7cba420598fa2a38b2ce01fc9737cba5c03eaa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 21:42:43 +1000 Subject: [PATCH 47/60] ENH: trailing space as per discussion on gh-303 --- config/filter.d/wuftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 5c1cbb58..de98d02d 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -18,7 +18,7 @@ _daemon = wu-ftpd # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = ^%(__prefix_line)sfailed login from \S+ \[\]$ +failregex = ^%(__prefix_line)sfailed login from \S+ \[\]\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From c44328b1a3922fbdb255f6f66282547ea2621e07 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 22:04:55 +1000 Subject: [PATCH 48/60] ENH: new "realm mismatch" message from https://issues.apache.org/bugzilla/show_bug.cgi?id=55284#c8 --- config/filter.d/apache-auth.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 2b1397cf..811017e3 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -44,6 +44,7 @@ failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server config ^%(_apache_error_client)s (AH01777: )?invalid nonce .* received - user attempted time travel\s*$ ^%(_apache_error_client)s (AH01778: )?user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$ + ^%(_apache_error_client)s (AH02486: )?realm mismatch - got `.*' but no realm specified\s*$ From aad7d08451a87465553fe08cc5d28d8805c65d51 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 20 Aug 2013 07:33:35 +1000 Subject: [PATCH 49/60] BF: disable filter expressions without tests --- config/filter.d/apache-auth.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 811017e3..7adb0fcf 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -28,6 +28,11 @@ before = apache-common.conf # to return the actual failure. # # See also: http://wiki.apache.org/httpd/ListOfErrors +# Expressions that don't have tests and aren't common. +# more be added with https://issues.apache.org/bugzilla/show_bug.cgi?id=55284 +# ^%(_apache_error_client)s (AH01778: )?user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ +# ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$ +# ^%(_apache_error_client)s (AH02486: )?realm mismatch - got `.*' but no realm specified\s*$ # failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s (AH01617: )?user .* authentication failure for "\S*": Password Mismatch$ @@ -42,10 +47,6 @@ failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server config ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*' received: \S*\s*$ ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*$ ^%(_apache_error_client)s (AH01777: )?invalid nonce .* received - user attempted time travel\s*$ - ^%(_apache_error_client)s (AH01778: )?user .*: nonce expired \([\d.]+ seconds old - max lifetime [\d.]+\) - sending new nonce\s*$ - ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$ - ^%(_apache_error_client)s (AH02486: )?realm mismatch - got `.*' but no realm specified\s*$ - # Option: ignoreregex From 0204cec5ce98ed3afaa3435cc230cba93d214ea9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 17:06:10 +1000 Subject: [PATCH 50/60] TST: www.example.com DNS changed --- testcases/files/testcase-usedns.log | 2 +- testcases/filtertestcase.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/testcases/files/testcase-usedns.log b/testcases/files/testcase-usedns.log index 87c99fdb..a91fd7ac 100644 --- a/testcases/files/testcase-usedns.log +++ b/testcases/files/testcase-usedns.log @@ -1,2 +1,2 @@ Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2 -Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2 +Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index d918bbf7..b961d0dc 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -632,12 +632,12 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) - output_yes = ('192.0.43.10', 2, 1124013539.0, + output_yes = ('93.184.216.119', 2, 1124013539.0, ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', - 'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) + 'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) - output_no = ('192.0.43.10', 1, 1124013539.0, - ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) + output_no = ('93.184.216.119', 1, 1124013539.0, + ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) # Actually no exception would be raised -- it will be just set to 'no' #self.assertRaises(ValueError, @@ -684,9 +684,9 @@ class DNSUtilsTests(unittest.TestCase): res = DNSUtils.textToIp('www.example.com', 'no') self.assertEqual(res, []) res = DNSUtils.textToIp('www.example.com', 'warn') - self.assertEqual(res, ['192.0.43.10']) + self.assertEqual(res, ['93.184.216.119']) res = DNSUtils.textToIp('www.example.com', 'yes') - self.assertEqual(res, ['192.0.43.10']) + self.assertEqual(res, ['93.184.216.119']) def testTextToIp(self): # Test hostnames @@ -698,7 +698,7 @@ class DNSUtilsTests(unittest.TestCase): for s in hostnames: res = DNSUtils.textToIp(s, 'yes') if s == 'www.example.com': - self.assertEqual(res, ['192.0.43.10']) + self.assertEqual(res, ['93.184.216.119']) else: self.assertEqual(res, []) From 1d9702be323eab8232ecf60293e314e8c0fb4393 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 17:34:36 +1000 Subject: [PATCH 51/60] TST: datetime mismatch to show error line --- testcases/samplestestcase.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 54afc864..e5a7723f 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -110,10 +110,12 @@ def testSampleRegexsFactory(name): # Verify timestamp and host as expected failregex, host, time = ret[0] self.assertEqual(host, faildata.get("host", None)) - self.assertEqual( - datetime.datetime.fromtimestamp(time), - datetime.datetime.strptime( - faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")) + fail2banTime = datetime.datetime.fromtimestamp(time) + jsonTime = datetime.datetime.strptime( + faildata.get("time", None), "%Y-%m-%dT%H:%M:%S") + + self.assertEqual(fail2banTime, jsonTime, + "Time mismatch %s != %s on %s:" % ( fail2banTime, jsonTime, line ) ) regexsUsed.add(failregex) From 21914d155e950abcba5773263c23ae2fea1794dd Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 17:49:09 +1000 Subject: [PATCH 52/60] TST: add failJSON data --- testcases/files/logs/apache-auth | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 7714c4bb..b2aa75ff 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -4,7 +4,10 @@ # should match # from https://github.com/fail2ban/fail2ban/issues/286 +# failJSON: { "time": "2013-07-11T01:21:41", "match": true , "host": "194.228.20.113" } [Thu Jul 11 01:21:41 2013] [error] [client 194.228.20.113] user not found: / + +# failJSON: { "time": "2013-07-11T01:21:43", "match": true , "host": "194.228.20.113" } [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: / # The failures below use the configuration described in testcases/files/config/apache-auth @@ -109,5 +112,5 @@ # failJSON: { "time": "2013-07-29T02:12:55", "match": true , "host": "127.0.0.1" } [Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel -# failJSON: { "time": "2005-06-01T02:17:42", "match": true , "host": "192.168.0.2" } +# failJSON: { "time": "2013-06-01T02:17:42", "match": true , "host": "192.168.0.2" } [Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found From c84a2e595a80390916a1981f71c775c2906b3080 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 24 Aug 2013 22:55:10 -0400 Subject: [PATCH 53/60] ENH(BF): put 'standard' template after more detailed ones with day of week and year otherwise years present in the freshly contributed by Dan apache regexes do not match although should have. I had also to adjust failing now vsftpd test --- ChangeLog | 2 ++ server/datedetector.py | 12 ++++++------ testcases/files/logs/vsftpd | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index edb30055..f06e46a9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -57,6 +57,8 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests enabled jail. Closes gh-63 * is now enforced to end with an alphanumeric * filter.d/roundcube-auth.conf -- anchored version + * date matching - for standard asctime formats prefer more detailed + first (thus use year if available) Alexander Dietrich * action.d/sendmail-common.conf -- added common sendmail settings file and made the sender display name configurable diff --git a/server/datedetector.py b/server/datedetector.py index 0ed9e00a..0f0feb6d 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -46,12 +46,6 @@ class DateDetector: def addDefaultTemplate(self): self.__lock.acquire() try: - # standard - template = DateStrptime() - template.setName("MONTH Day Hour:Minute:Second") - template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") - template.setPattern("%b %d %H:%M:%S") - self._appendTemplate(template) # asctime template = DateStrptime() template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year") @@ -64,6 +58,12 @@ class DateDetector: template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") template.setPattern("%a %b %d %H:%M:%S") self._appendTemplate(template) + # standard - most loose from above 3 so by default follows after + template = DateStrptime() + template.setName("MONTH Day Hour:Minute:Second") + template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") + template.setPattern("%b %d %H:%M:%S") + self._appendTemplate(template) # simple date template = DateStrptime() template.setName("Year/Month/Day Hour:Minute:Second") diff --git a/testcases/files/logs/vsftpd b/testcases/files/logs/vsftpd index f3fb997f..ac6d0454 100644 --- a/testcases/files/logs/vsftpd +++ b/testcases/files/logs/vsftpd @@ -5,6 +5,6 @@ Oct 11 01:06:47 ServerJV vsftpd: (pam_unix) authentication failure; logname= uid Feb 6 12:02:29 server vsftpd(pam_unix)[15522]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=64.168.103.1 user=user1 #2 Internal -# failJSON: { "time": "2005-01-19T12:20:33", "match": true , "host": "64.106.46.98" } +# failJSON: { "time": "2007-01-19T12:20:33", "match": true , "host": "64.106.46.98" } Fri Jan 19 12:20:33 2007 [pid 27202] [anonymous] FAIL LOGIN: Client "64.106.46.98" From 8e467437b251f94c818cb280012c7030b60081af Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 18:09:39 +1000 Subject: [PATCH 54/60] TST: fix year on asctime --- testcases/files/logs/apache-nohome | 4 ++-- testcases/files/logs/apache-noscript | 4 ++-- testcases/files/logs/apache-overflows | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testcases/files/logs/apache-nohome b/testcases/files/logs/apache-nohome index aea0d816..78327a62 100644 --- a/testcases/files/logs/apache-nohome +++ b/testcases/files/logs/apache-nohome @@ -1,6 +1,6 @@ # Apache 2.2 -# failJSON: { "time": "2005-06-01T11:23:08", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2013-06-01T11:23:08", "match": true , "host": "1.2.3.4" } [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] File does not exist: /xxx/~ # Apache 2.4 -# failJSON: { "time": "2005-06-27T11:55:44", "match": true , "host": "192.0.2.12" } +# failJSON: { "time": "2013-06-27T11:55:44", "match": true , "host": "192.0.2.12" } [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 192.0.2.12:46652] AH00128: File does not exist: /xxx/~ diff --git a/testcases/files/logs/apache-noscript b/testcases/files/logs/apache-noscript index 172bb510..53e33baf 100644 --- a/testcases/files/logs/apache-noscript +++ b/testcases/files/logs/apache-noscript @@ -1,4 +1,4 @@ -# failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" } +# failJSON: { "time": "2013-06-09T07:57:47", "match": true , "host": "192.0.43.10" } [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat -# failJSON: { "time": "2005-07-22T06:48:30", "match": true , "host": "198.51.100.86" } +# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" } [Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 1af377f1..d40c1c4f 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,4 +1,4 @@ -# failJSON: { "time": "2005-03-16T15:39:29", "match": true , "host": "58.179.109.179" } +# failJSON: { "time": "2010-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 -# failJSON: { "time": "2005-03-15T15:44:47", "match": true , "host": "121.222.2.133" } +# failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 From cb61fcd326e727f1b8ec5c2787cd18b8fab50506 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 18:11:54 +1000 Subject: [PATCH 55/60] TST: standardise output format on Time mismatch test --- testcases/samplestestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index e5a7723f..01a78f6c 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -115,7 +115,8 @@ def testSampleRegexsFactory(name): faildata.get("time", None), "%Y-%m-%dT%H:%M:%S") self.assertEqual(fail2banTime, jsonTime, - "Time mismatch %s != %s on %s:" % ( fail2banTime, jsonTime, line ) ) + "Time mismatch %s != %s on: %s:%i %r:" % + (fail2banTime, jsonTime, logFile.filename(), logFile.filelineno(), line ) ) regexsUsed.add(failregex) From 6a567276691f0d1b2b4fcdc2587691e639f4be72 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 18:30:30 +1000 Subject: [PATCH 56/60] BF: apache-common regex - datetime could be entirely consumed --- config/filter.d/apache-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index cc35ae5f..69364247 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -18,4 +18,4 @@ after = apache-common.local # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] # 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652] # Reference: https://github.com/fail2ban/fail2ban/issues/268 -_apache_error_client = \[[^]]+\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\]( \S+:)? +_apache_error_client = \[[^]]*\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\]( \S+:)? From 9a1df3501b936d730e330ecb878802cf93c59aed Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 20:19:42 +1000 Subject: [PATCH 57/60] TST: display details of duplicate matches --- testcases/samplestestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 01a78f6c..938b9c13 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -106,7 +106,8 @@ def testSampleRegexsFactory(name): self.assertTrue(faildata.get('match', False), "Line matched when shouldn't have: %s:%i %r" % (logFile.filename(), logFile.filelineno(), line)) - self.assertEqual(len(ret), 1, "Multiple regexs matched") + self.assertEqual(len(ret), 1, "Multiple regexs matched - %s:%i %r " % + (logFile.filename(), logFile.filelineno(), ret)) # Verify timestamp and host as expected failregex, host, time = ret[0] self.assertEqual(host, faildata.get("host", None)) From 62c13c15d6b4923d7b0c2611d2d8144c2402af8d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 21:02:30 +1000 Subject: [PATCH 58/60] TST: reorder and condense error message for Multiple regexs matched --- testcases/samplestestcase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 938b9c13..c7ff0b9a 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -106,8 +106,9 @@ def testSampleRegexsFactory(name): self.assertTrue(faildata.get('match', False), "Line matched when shouldn't have: %s:%i %r" % (logFile.filename(), logFile.filelineno(), line)) - self.assertEqual(len(ret), 1, "Multiple regexs matched - %s:%i %r " % - (logFile.filename(), logFile.filelineno(), ret)) + self.assertEqual(len(ret), 1, "Multiple regexs matched %r - %s:%i" % + (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno())) + # Verify timestamp and host as expected failregex, host, time = ret[0] self.assertEqual(host, faildata.get("host", None)) From ced271b908d13f35951a7997bb0b5645eb21191e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 21:11:47 +1000 Subject: [PATCH 59/60] ENH: date for apache-2.4 - adds milliseconds --- ChangeLog | 1 + server/datedetector.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f06e46a9..d088833c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Daniel Black * filter.d/exim-spam.conf -- a splitout of exim's spam regexes with additions for greater control over filtering spam. + * add date expression for apache-2.4 - milliseconds - Enhancements: Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening diff --git a/server/datedetector.py b/server/datedetector.py index 0f0feb6d..83d69733 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -46,7 +46,13 @@ class DateDetector: def addDefaultTemplate(self): self.__lock.acquire() try: - # asctime + # asctime with subsecond + template = DateStrptime() + template.setName("WEEKDAY MONTH Day Hour:Minute:Second[.subsecond] Year") + template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}\.\d+ \d{4}") + template.setPattern("%a %b %d %H:%M:%S.%f %Y") + self._appendTemplate(template) + # asctime without no subsecond template = DateStrptime() template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year") template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}") From fd7cc5bda75625052fd4fbf3f7e68f00bd3fa0f9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 25 Aug 2013 21:13:11 +1000 Subject: [PATCH 60/60] BF: duplicate regex match fixed --- config/filter.d/apache-auth.conf | 6 +++--- config/filter.d/apache-common.conf | 2 +- config/filter.d/apache-nohome.conf | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 7adb0fcf..fa828b72 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -36,17 +36,17 @@ before = apache-common.conf # failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*\s*$ ^%(_apache_error_client)s (AH01617: )?user .* authentication failure for "\S*": Password Mismatch$ - ^%(_apache_error_client)s (AH01618: )?user .* not found: \S*\s*$ + ^%(_apache_error_client)s (AH01618: )?user .* not found(: )?\S*\s*$ ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*\s*$ ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*: password mismatch: \S*\s*$ - ^%(_apache_error_client)s (AH0179[01]: )?(Digest: )?user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ + ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*' in realm `.+' (not found|denied by provider): \S*\s*$ ^%(_apache_error_client)s (AH01631: )?user .*: authorization failure for "\S*":\s*$ ^%(_apache_error_client)s (AH0177[56]: )?(Digest: )?invalid nonce .* received - (length|hash) is not \S+\s*$ ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*' but expected `.+'\s*$ ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*' received: \S*\s*$ ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*$ - ^%(_apache_error_client)s (AH01777: )?invalid nonce .* received - user attempted time travel\s*$ + ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .* received - user attempted time travel\s*$ # Option: ignoreregex diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index 69364247..134fad29 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -18,4 +18,4 @@ after = apache-common.local # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] # 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652] # Reference: https://github.com/fail2ban/fail2ban/issues/268 -_apache_error_client = \[[^]]*\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\]( \S+:)? +_apache_error_client = \[[^]]*\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\] diff --git a/config/filter.d/apache-nohome.conf b/config/filter.d/apache-nohome.conf index 1347b10d..0eede317 100644 --- a/config/filter.d/apache-nohome.conf +++ b/config/filter.d/apache-nohome.conf @@ -19,7 +19,7 @@ before = apache-common.conf # per-domain log files. # Values: TEXT # -failregex = ^%(_apache_error_client)s File does not exist: .*/~.* +failregex = ^%(_apache_error_client)s (AH00128: )?File does not exist: .*/~.* # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored.