Merge pull request #940 from leeclemens/ENH/ApacheFakeGoogleBot

New jail: apache-fakegooglebot

ChangeLog

@@ -36,6 +36,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
 - New Features:
    - New filter:
      - postfix-rbl  Thanks Lee Clemens
+     - apache-fakegooglebot.conf  Thanks Lee Clemens
    - New recursive embedded substitution feature added:
      - `<<PREF>HOST>` becomes `<IPV4HOST>` for PREF=`IPV4`;
      - `<<PREF>HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`;

MANIFEST

@@ -266,6 +266,8 @@ config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
 config/filter.d/horde.conf
+config/filter.d/ignorecommands
+config/filter.d/ignorecommands/apache-fakegooglebot
 config/filter.d/kerio.conf
 config/filter.d/lighttpd-auth.conf
 config/filter.d/monit.conf

MANIFEST.in

@@ -2,3 +2,4 @@ include ChangeLog COPYING DEVELOP FILTERS README.* THANKS TODO CONTRIBUTING* Vag
 graft doc
 graft files
 recursive-include config *.conf *.py
+recursive-include config/filter.d/ignorecommands *

config/filter.d/apache-fakegooglebot.conf

@@ -0,0 +1,14 @@
+# Fail2Ban filter for fake Googlebot User Agents
+
+[Definition]
+
+failregex = ^<HOST> .*Googlebot.*$
+
+ignoreregex =
+
+
+# DEV Notes:
+#
+# Author: Lee Clemens
+# Thanks: Johannes B. Ullrich, Ph.D.
+# Reference: https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/

config/jail.conf

@@ -277,6 +277,14 @@ logpath  = %(apache_error_log)s
 maxretry = 2
 
 
+[apache-fakegooglebot]
+
+port     = http,https
+logpath  = %(apache_access_log)s
+maxretry = 1
+ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot <ip>
+
+
 [apache-modsecurity]
 
 port     = http,https

config/paths-common.conf

@@ -61,3 +61,6 @@ dovecot_log = %(syslog_mail_warn)s
 solidpop3d_log = %(syslog_local0)s
 
 mysql_log = %(syslog_daemon)s
+
+# Directory with ignorecommand scripts
+ignorecommands_dir = /etc/fail2ban/filter.d/ignorecommands

fail2ban/client/filterreader.py

@@ -71,7 +71,7 @@ class FilterReader(DefinitionInitConfigReader):
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self._jailName, "addignoreregex", regex])		
+						stream.append(["set", self._jailName, "addignoreregex", regex])
 		if self._initOpts:
 			if 'maxlines' in self._initOpts:
 				# We warn when multiline regex is used without maxlines > 1

fail2ban/tests/files/logs/apache-fakegooglebot

@@ -0,0 +1,5 @@
+# Apache 2.2
+# failJSON: { "time": "2015-01-31T14:29:44", "match": true, "host": "66.249.66.1" }
+66.249.66.1 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + 293 1149 546
+# failJSON: { "time": "2015-01-31T14:29:44", "match": false, "host": "93.184.216.34" }
+93.184.216.34 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "NOT A __GOOGLE_BOT__" + 293 1149 546

fail2ban/tests/filtertestcase.py

@@ -1011,6 +1011,12 @@ class DNSUtilsTests(unittest.TestCase):
 			else:
 				self.assertEqual(res, [])
 
+	def testIpToName(self):
+		res = DNSUtils.ipToName('66.249.66.1')
+		self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com')
+		res = DNSUtils.ipToName('10.0.0.0')
+		self.assertEqual(res, None)
+
 class JailTests(unittest.TestCase):
 
 	def testSetBackend_gh83(self):

fail2ban/tests/samplestestcase.py

@@ -141,7 +141,8 @@ def testSampleRegexsFactory(name):
 
 	return testFilter
 
-for filter_ in filter(lambda x: not x.endswith('common.conf'), os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
+for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'),
+					  os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
 	filterName = filter_.rpartition(".")[0]
 	if not filterName.startswith('.'):
 		setattr(

setup.py

@@ -124,6 +124,9 @@ setup(
 		('/etc/fail2ban/filter.d',
 			glob("config/filter.d/*.conf")
 		),
+		('/etc/fail2ban/filter.d/ignorecommands',
+			glob("config/filter.d/ignorecommands/*")
+		),
 		('/etc/fail2ban/action.d',
 			glob("config/action.d/*.conf") +
 			glob("config/action.d/*.py")