Merge pull request #557 from grooverdan/apache-botsearch

ENH: Apache botsearch + BF: tag substition

ChangeLog

@@ -55,6 +55,7 @@ configuration before relying on it.
    * Added action xarf-login-attack to report formatted attack messages
      according to the XARF standard (v0.2). Close gh-105
    * Support PyPy
+   * Add filter for apache-botsearch
    * Filter for stunnel
    * Filter for Counter Strike 1.6. Thanks to onorua for logs.
      Close gh-347

MANIFEST

@@ -102,6 +102,7 @@ fail2ban/tests/files/logs/bsd/syslog-vv.txt
 fail2ban/tests/files/logs/3proxy
 fail2ban/tests/files/logs/apache-auth
 fail2ban/tests/files/logs/apache-badbots
+fail2ban/tests/files/logs/apache-botscripts
 fail2ban/tests/files/logs/apache-modsecurity
 fail2ban/tests/files/logs/apache-nohome
 fail2ban/tests/files/logs/apache-noscript
@@ -170,6 +171,7 @@ config/fail2ban.conf
 config/filter.d/common.conf
 config/filter.d/apache-auth.conf
 config/filter.d/apache-badbots.conf
+config/filter.d/apache-botsearch.conf
 config/filter.d/apache-nohome.conf
 config/filter.d/apache-noscript.conf
 config/filter.d/apache-overflows.conf

config/filter.d/apache-botsearch.conf

@@ -0,0 +1,48 @@
+# Fail2Ban filter to match web requests for selected URLs that don't exist
+#
+# This filter is aimed at blocking specific URLs that don't exist. This
+# could be a set of URLs places in a Disallow: directive in robots.txt or
+# just some web services that don't exist caused bots are searching for
+# exploitable content. This filter is designed to have a low false postitive
+# rate due.
+#
+# An alternative to this is the apache-noscript filter which blocks all
+# types of scripts that don't exist.
+#
+#
+# This is normally a predefined list of exploitable or valuable web services
+# that are hidden or aren't actually installed.
+#
+
+[INCLUDES]
+
+# overwrite with apache-common.local if _apache_error_client is incorrect.
+before = apache-common.conf
+
+[Definition]
+
+failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): <webroot><block>(, referer: \S+)?\s*$
+            ^%(_apache_error_client)s script '<webroot><block>' not found or unable to stat(, referer: \S+)?\s*$
+
+ignoreregex = 
+
+
+[Init]
+
+# Webroot represents the webroot on which all other files are based
+webroot = /var/www/
+# Block is the actual non-found directories to block
+block = (<webmail>|<phpmyadmin>|<wordpress>)[^,]*
+
+# These are just convient definitions that assist the blocking of stuff that 
+# isn't installed
+webmail = roundcube|(ext)?mail|horde|(v-?)?webmail
+
+phpmyadmin = (typo3/|xampp/|admin/|)(pma|(php)?[Mm]y[Aa]dmin)
+
+wordpress = wp-(login|signup)\.php
+
+
+# DEV Notes:
+#
+# Author: Daniel Black

config/filter.d/apache-noscript.conf

@@ -1,5 +1,13 @@
 # Fail2Ban filter to block web requests for scripts (on non scripted websites)
 #
+# This matches many types of scripts that don't exist. This could generate a
+# lot of false positive matches in cases like wikis and forums where users
+# no affiliated with the website can insert links to missing files/scripts into
+# pages and cause non-malicious browsers of the site to trigger against this
+# filter.
+#
+# If you'd like to match specific URLs that don't exist see the
+# apache-botsearch filter.
 #
 
 [INCLUDES]
@@ -19,6 +27,6 @@ ignoreregex =
 #
 # https://wiki.apache.org/httpd/ListOfErrors for apache error IDs
 #
-# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is Before http-2.2
+# Second regex, script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$ is in httpd-2.2
 #
 # Author: Cyril Jaquier

config/jail.conf

@@ -355,6 +355,13 @@ logpath  = /var/log/apache*/*error.log
 maxretry = 2
 
 
+[apache-botsearch]
+
+port     = http,https
+logpath  = /var/log/apache*/*error.log
+maxretry = 2
+
+
 [apache-modsecurity]
 
 port     = http,https

fail2ban/server/action.py

@@ -379,11 +379,11 @@ class CommandAction(ActionBase):
 				#logSys.log(5, 'found: %s' % found_tag)
 				if found_tag == tag or found_tag in done:
 					# recursive definitions are bad
-					#logSys.log(5, 'recursion fail')
+					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 					return False
 				else:
 					if tags.has_key(found_tag):
-						value = value[0:m.start()] + tags[found_tag] + value[m.end():]
+						value = value.replace('<%s>' % found_tag , tags[found_tag])
 						#logSys.log(5, 'value now: %s' % value)
 						done.append(found_tag)
 						m = t.search(value, m.start())

fail2ban/tests/actiontestcase.py

@@ -60,6 +60,12 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
+		# Multiple stuff on same line is ok
+		self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP> evilperson=<honeypot>', 'honeypot': 'pokie', 'ignoreregex': ''}),
+								{ 'failregex': "to=pokie fromip=<IP> evilperson=pokie",
+									'honeypot': 'pokie',
+									'ignoreregex': '',
+								})
 		# rest is just cool
 		self.assertEqual(CommandAction.substituteRecursiveTags(aInfo),
 								{ 'HOST': "192.0.2.0",

fail2ban/tests/files/logs/apache-botsearch

@@ -0,0 +1,43 @@
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /var/www/wp-login.php
+
+# failJSON: { "time": "2013-12-23T09:49:10", "match": true , "host": "115.249.248.145" }
+[Mon Dec 23 09:49:10 2013] [error] [client 115.249.248.145] File does not exist: /var/www/pma
+# failJSON: { "time": "2013-12-23T09:49:10", "match": true , "host": "115.249.248.145" }
+[Mon Dec 23 09:49:10 2013] [error] [client 115.249.248.145] File does not exist: /var/www/phpmyadmin
+# failJSON: { "time": "2013-12-23T09:49:13", "match": true , "host": "115.249.248.145" }
+[Mon Dec 23 09:49:13 2013] [error] [client 115.249.248.145] File does not exist: /var/www/webmail
+# failJSON: { "time": "2013-12-23T09:49:13", "match": true , "host": "115.249.248.145" }
+[Mon Dec 23 09:49:13 2013] [error] [client 115.249.248.145] File does not exist: /var/www/mail
+
+# failJSON: { "time": "2013-12-31T09:13:47", "match": true , "host": "176.102.37.56" }
+[Tue Dec 31 09:13:47 2013] [error] [client 176.102.37.56] script '/var/www/wp-login.php' not found or unable to stat
+
+# failJSON: { "time": "2014-01-03T09:20:23", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:20:23 2014] [error] [client 46.23.77.174] File does not exist: /var/www/mail
+# failJSON: { "time": "2014-01-03T09:20:25", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:20:25 2014] [error] [client 46.23.77.174] File does not exist: /var/www/mail_this_entry
+# failJSON: { "time": "2014-01-03T09:26:52", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:26:52 2014] [error] [client 46.23.77.174] File does not exist: /var/www/pmapper-3.2-beta3
+# failJSON: { "time": "2014-01-03T09:33:53", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:33:53 2014] [error] [client 46.23.77.174] File does not exist: /var/www/v-webmail
+# failJSON: { "time": "2014-01-03T09:34:15", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:34:15 2014] [error] [client 46.23.77.174] File does not exist: /var/www/vwebmail
+# failJSON: { "time": "2014-01-03T09:35:47", "match": true , "host": "46.23.77.174" }
+[Fri Jan 03 09:35:47 2014] [error] [client 46.23.77.174] File does not exist: /var/www/webmail
+# failJSON: { "time": "2013-12-23T21:21:39", "match": true , "host": "183.60.244.49" }
+[Mon Dec 23 21:21:39 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com
+# failJSON: { "time": "2013-12-23T21:21:44", "match": true , "host": "183.60.244.49" }
+[Mon Dec 23 21:21:44 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com
+# failJSON: { "time": "2013-12-23T21:21:47", "match": true , "host": "183.60.244.49" }
+[Mon Dec 23 21:21:47 2013] [error] [client 183.60.244.49] File does not exist: /var/www/mails, referer: http://www.baidu.com
+# failJSON: { "time": "2013-12-23T21:22:00", "match": true , "host": "183.60.244.49" }
+[Mon Dec 23 21:22:00 2013] [error] [client 183.60.244.49] File does not exist: /var/www/extmail, referer: http://www.baidu.com
+# failJSON: { "time": "2013-12-23T21:22:16", "match": true , "host": "183.60.244.49" }
+[Mon Dec 23 21:22:16 2013] [error] [client 183.60.244.49] File does not exist: /var/www/phpmyadmin, referer: http://www.baidu.com
+
+# failJSON: { "time": "2014-01-03T14:50:39", "match": false , "host": "92.43.20.165" }
+[Fri Jan 03 14:50:39 2014] [error] [client 92.43.20.165] script '/var/www/forum/mail.php' not found or unable to stat
+
+# failJSON: { "time": "2014-12-06T09:29:34", "match": false , "host": "122.49.201.178" }
+[Fri Dec 06 09:29:34 2013] [error] [client 122.49.201.178] client denied by server configuration: /var/www/webmail/.htaccess