From 7c09a61ca5efccacf1e505ef5297b87468950468 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 3 Jan 2014 23:12:58 +1100 Subject: [PATCH 1/7] ENH: add apache-botsearch. Closes gh-544 --- ChangeLog | 1 + MANIFEST | 2 ++ config/filter.d/apache-botsearch.conf | 37 ++++++++++++++++++++++ fail2ban/tests/files/logs/apache-botsearch | 2 ++ 4 files changed, 42 insertions(+) create mode 100644 config/filter.d/apache-botsearch.conf create mode 100644 fail2ban/tests/files/logs/apache-botsearch diff --git a/ChangeLog b/ChangeLog index b2a98046..60a66ee6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -56,6 +56,7 @@ configuration before relying on it. according to the XARF standard (v0.2). Close gh-105 * Add filter for apache-modsecurity * Support PyPy + * Add filter for apache-botsearch - Enhancements * Multiline filter for sendmail-spam. Close gh-418 diff --git a/MANIFEST b/MANIFEST index b03d309f..47dc4d1c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -96,6 +96,7 @@ fail2ban/tests/files/logs/bsd/syslog-vv.txt fail2ban/tests/files/logs/3proxy fail2ban/tests/files/logs/apache-auth fail2ban/tests/files/logs/apache-badbots +fail2ban/tests/files/logs/apache-botscripts fail2ban/tests/files/logs/apache-modsecurity fail2ban/tests/files/logs/apache-nohome fail2ban/tests/files/logs/apache-noscript @@ -159,6 +160,7 @@ config/jail.conf config/filter.d/common.conf config/filter.d/apache-auth.conf config/filter.d/apache-badbots.conf +config/filter.d/apache-botsearch.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf new file mode 100644 index 00000000..f3bb6e70 --- /dev/null +++ b/config/filter.d/apache-botsearch.conf @@ -0,0 +1,37 @@ +# Fail2Ban filter to block web requests for stuff that doesn't exist +# +# This is normally a predefined list of exploitable or valuable web services +# that are hidden or aren't actually installed. +# + +[INCLUDES] + +# overwrite with apache-common.local if _apache_error_client is incorrect. +before = apache-common.conf + +[Definition] + +failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): (, referer: \S+)?\s*$ + +ignoreregex = + + +[Init] + +# Webroot represents the webroot on which all other files are based +webroot = /var/www/ +# Block is the actual non-found directories to block +block = (||) + +# These are just convient definitions that assist the blocking of stuff that +# isn't installed +webmail = roundcube|mail|horde|webmail + +phpmyadmin = (typo3/|xampp/|)(pma|(php)?myadmin) + +wordpress = wp-(login|signup)\.php + + +# DEV Notes: +# +# Author: Daniel Black diff --git a/fail2ban/tests/files/logs/apache-botsearch b/fail2ban/tests/files/logs/apache-botsearch new file mode 100644 index 00000000..476f3ad2 --- /dev/null +++ b/fail2ban/tests/files/logs/apache-botsearch @@ -0,0 +1,2 @@ +# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" } +[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /var/www/wp-login.php From 6ce2ba289548e1da5decda538452f00e29ab7309 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 5 Jan 2014 11:48:35 +1100 Subject: [PATCH 2/7] ENH: additional phpmyadmin tips from Tom on http://www.fail2ban.org/wiki/index.php?title=Fail2ban:Community_Portal. Block is now a prefix of a path --- config/filter.d/apache-botsearch.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf index f3bb6e70..95df97ca 100644 --- a/config/filter.d/apache-botsearch.conf +++ b/config/filter.d/apache-botsearch.conf @@ -21,13 +21,13 @@ ignoreregex = # Webroot represents the webroot on which all other files are based webroot = /var/www/ # Block is the actual non-found directories to block -block = (||) +block = (||)[^,]* # These are just convient definitions that assist the blocking of stuff that # isn't installed webmail = roundcube|mail|horde|webmail -phpmyadmin = (typo3/|xampp/|)(pma|(php)?myadmin) +phpmyadmin = (typo3/|xampp/|admin/|)(pma|(php)?[Mm]y[Aa]dmin) wordpress = wp-(login|signup)\.php From 809581ae9957181e07f4a8efc4843b09682d041a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 7 Jan 2014 11:52:21 +1100 Subject: [PATCH 3/7] ENH: jail.conf for apache-botsearch --- config/jail.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index acec2c5b..9083c788 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -348,6 +348,13 @@ logpath = /var/log/apache*/*error.log maxretry = 2 +[apache-botsearch] + +port = http,https +logpath = /var/log/apache*/*error.log +maxretry = 2 + + [apache-modsecurity] port = http,https From 8e5366a7e9bba1ff013f88b89fd84882b975b814 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 10 Jan 2014 07:34:01 +1100 Subject: [PATCH 4/7] DOC: for apache-botsearch and apache-botsearch --- config/filter.d/apache-botsearch.conf | 12 +++++++++++- config/filter.d/apache-noscript.conf | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf index 95df97ca..3f887dc5 100644 --- a/config/filter.d/apache-botsearch.conf +++ b/config/filter.d/apache-botsearch.conf @@ -1,4 +1,14 @@ -# Fail2Ban filter to block web requests for stuff that doesn't exist +# Fail2Ban filter to match web requests for selected URLs that don't exist +# +# This filter is aimed at blocking specific URLs that don't exist. This +# could be a set of URLs places in a Disallow: directive in robots.txt or +# just some web services that don't exist caused bots are searching for +# exploitable content. This filter is designed to have a low false postitive +# rate due. +# +# An alternative to this is the apache-noscript filter which blocks all +# types of scripts that don't exist. +# # # This is normally a predefined list of exploitable or valuable web services # that are hidden or aren't actually installed. diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 9a591ca3..d0f130f6 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -1,5 +1,13 @@ # Fail2Ban filter to block web requests for scripts (on non scripted websites) # +# This matches many types of scripts that don't exist. This could generate a +# lot of false positive matches in cases like wikis and forums where users +# no affiliated with the website can insert links to missing files/scripts into +# pages and cause non-malicious browsers of the site to trigger against this +# filter. +# +# If you'd like to match specific URLs that don't exist see the +# apache-botsearch filter. # [INCLUDES] From 4b33f96db4e686020a64a9babe45d93c261887e7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 10 Jan 2014 08:35:37 +1100 Subject: [PATCH 5/7] DOC: fix comment regarding apache version in apache-noscript --- config/filter.d/apache-noscript.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index d0f130f6..fbc1af64 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -27,6 +27,6 @@ ignoreregex = # # https://wiki.apache.org/httpd/ListOfErrors for apache error IDs # -# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is Before http-2.2 +# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is in httpd-2.2 # # Author: Cyril Jaquier From 9e358541b7f3ed64c70a8500aa141941f65d465e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 10 Jan 2014 08:39:39 +1100 Subject: [PATCH 6/7] BF: fix multiple tag substitutions on the same line --- fail2ban/server/action.py | 4 ++-- fail2ban/tests/actiontestcase.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 77b97fcd..94beb784 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -379,11 +379,11 @@ class CommandAction(ActionBase): #logSys.log(5, 'found: %s' % found_tag) if found_tag == tag or found_tag in done: # recursive definitions are bad - #logSys.log(5, 'recursion fail') + #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False else: if tags.has_key(found_tag): - value = value[0:m.start()] + tags[found_tag] + value[m.end():] + value = value.replace('<%s>' % found_tag , tags[found_tag]) #logSys.log(5, 'value now: %s' % value) done.append(found_tag) m = t.search(value, m.start()) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 897813b2..35de36dd 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -60,6 +60,12 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(CommandAction.substituteRecursiveTags({'A': ''}), {'A': ''}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + # Multiple stuff on same line is ok + self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), + { 'failregex': "to=pokie fromip= evilperson=pokie", + 'honeypot': 'pokie', + 'ignoreregex': '', + }) # rest is just cool self.assertEqual(CommandAction.substituteRecursiveTags(aInfo), { 'HOST': "192.0.2.0", From b0baab3a0ed67a24fe7e5be06abc5e1f9d624d3e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 10 Jan 2014 08:40:24 +1100 Subject: [PATCH 7/7] ENH: more test cases and wider regex --- config/filter.d/apache-botsearch.conf | 3 +- fail2ban/tests/files/logs/apache-botsearch | 41 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf index 3f887dc5..f7f5488c 100644 --- a/config/filter.d/apache-botsearch.conf +++ b/config/filter.d/apache-botsearch.conf @@ -22,6 +22,7 @@ before = apache-common.conf [Definition] failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): (, referer: \S+)?\s*$ + ^%(_apache_error_client)s script '' not found or unable to stat(, referer: \S+)?\s*$ ignoreregex = @@ -35,7 +36,7 @@ block = (||)[^,]* # These are just convient definitions that assist the blocking of stuff that # isn't installed -webmail = roundcube|mail|horde|webmail +webmail = roundcube|(ext)?mail|horde|(v-?)?webmail phpmyadmin = (typo3/|xampp/|admin/|)(pma|(php)?[Mm]y[Aa]dmin) diff --git a/fail2ban/tests/files/logs/apache-botsearch b/fail2ban/tests/files/logs/apache-botsearch index 476f3ad2..dc2de366 100644 --- a/fail2ban/tests/files/logs/apache-botsearch +++ b/fail2ban/tests/files/logs/apache-botsearch @@ -1,2 +1,43 @@ # failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" } [Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /var/www/wp-login.php + +# failJSON: { "time": "2013-12-23T09:49:10", "match": true , "host": "115.249.248.145" } +[Mon Dec 23 09:49:10 2013] [error] [client 115.249.248.145] File does not exist: /var/www/pma +# failJSON: { "time": "2013-12-23T09:49:10", "match": true , "host": "115.249.248.145" } +[Mon Dec 23 09:49:10 2013] [error] [client 115.249.248.145] File does not exist: /var/www/phpmyadmin +# failJSON: { "time": "2013-12-23T09:49:13", "match": true , "host": "115.249.248.145" } +[Mon Dec 23 09:49:13 2013] [error] [client 115.249.248.145] File does not exist: /var/www/webmail +# failJSON: { "time": "2013-12-23T09:49:13", "match": true , "host": "115.249.248.145" } +[Mon Dec 23 09:49:13 2013] [error] [client 115.249.248.145] File does not exist: /var/www/mail + +# failJSON: { "time": "2013-12-31T09:13:47", "match": true , "host": "176.102.37.56" } +[Tue Dec 31 09:13:47 2013] [error] [client 176.102.37.56] script '/var/www/wp-login.php' not found or unable to stat + +# failJSON: { "time": "2014-01-03T09:20:23", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:20:23 2014] [error] [client 46.23.77.174] File does not exist: /var/www/mail +# failJSON: { "time": "2014-01-03T09:20:25", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:20:25 2014] [error] [client 46.23.77.174] File does not exist: /var/www/mail_this_entry +# failJSON: { "time": "2014-01-03T09:26:52", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:26:52 2014] [error] [client 46.23.77.174] File does not exist: /var/www/pmapper-3.2-beta3 +# failJSON: { "time": "2014-01-03T09:33:53", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:33:53 2014] [error] [client 46.23.77.174] File does not exist: /var/www/v-webmail +# failJSON: { "time": "2014-01-03T09:34:15", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:34:15 2014] [error] [client 46.23.77.174] File does not exist: /var/www/vwebmail +# failJSON: { "time": "2014-01-03T09:35:47", "match": true , "host": "46.23.77.174" } +[Fri Jan 03 09:35:47 2014] [error] [client 46.23.77.174] File does not exist: /var/www/webmail +# failJSON: { "time": "2013-12-23T21:21:39", "match": true , "host": "183.60.244.49" } +[Mon Dec 23 21:21:39 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com +# failJSON: { "time": "2013-12-23T21:21:44", "match": true , "host": "183.60.244.49" } +[Mon Dec 23 21:21:44 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com +# failJSON: { "time": "2013-12-23T21:21:47", "match": true , "host": "183.60.244.49" } +[Mon Dec 23 21:21:47 2013] [error] [client 183.60.244.49] File does not exist: /var/www/mails, referer: http://www.baidu.com +# failJSON: { "time": "2013-12-23T21:22:00", "match": true , "host": "183.60.244.49" } +[Mon Dec 23 21:22:00 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com +# failJSON: { "time": "2013-12-23T21:22:16", "match": true , "host": "183.60.244.49" } +[Mon Dec 23 21:22:16 2013] [error] [client 183.60.244.49] File does not exist: /var/www/phpmyadmin, referer: http://www.baidu.com + +# failJSON: { "time": "2014-01-03T14:50:39", "match": false , "host": "92.43.20.165" } +[Fri Jan 03 14:50:39 2014] [error] [client 92.43.20.165] script '/var/www/forum/mail.php' not found or unable to stat + +# failJSON: { "time": "2014-12-06T09:29:34", "match": false , "host": "122.49.201.178" } +[Fri Dec 06 09:29:34 2013] [error] [client 122.49.201.178] client denied by server configuration: /var/www/webmail/.htaccess