New jail: apache-fakegooglebot

Detects fake googlebot user agents in apache access log

ChangeLog

@@ -33,6 +33,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
 - New Features:
    - New filter:
      - postfix-rbl  Thanks Lee Clemens
+     - apache-fakegooglebot.conf  Thanks Lee Clemens
    - New recursive embedded substitution feature added:
      - `<<PREF>HOST>` becomes `<IPV4HOST>` for PREF=`IPV4`;
      - `<<PREF>HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`;
@@ -49,6 +50,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
        - empty or "basic" works as-is
        - "cymru" additionally prints (ASN, Country RIR) per banned IP
        - Requires dnspython or dnspython3
+   - New ignorecommand:
+     - apache-fakegooglebot.sh  Thanks Lee Clemens
 
 - Enhancements:
    * Enable multiport for firewallcmd-new action.  Closes gh-834

config/filter.d/apache-fakegooglebot.conf

@@ -0,0 +1,14 @@
+# Fail2Ban filter for fake Googlebot User Agents
+
+[Definition]
+
+failregex = ^<HOST> .*Googlebot.*$
+
+ignoreregex =
+
+
+# DEV Notes:
+#
+# Author: Lee Clemens
+# Thanks: Johannes B. Ullrich, Ph.D.
+# Reference: https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/

config/filter.d/ignorecommands/apache-fakegooglebot.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Based on: https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
+
+if [ "$#" -ne 1 ]; then
+    echo "Unexpected number of arguments: $#"
+    exit 1
+else
+    b="$1"
+    h=$(host ${b})
+    if echo ${h} | grep -e ' crawl-.*\.googlebot\.com\.$'; then
+      h=$(echo ${h} | cut -f5 -d' ')
+      n=$(host ${h} | cut -f4 -d' ')
+      if [ "${n}" = "${b}" ] ; then
+        exit 0
+      else
+        exit 1
+      fi
+    else
+      exit 1
+    fi
+fi

config/jail.conf

@@ -277,6 +277,14 @@ logpath  = %(apache_error_log)s
 maxretry = 2
 
 
+[apache-fakegooglebot]
+
+port     = http,https
+logpath  = %(apache_access_log)s
+maxretry = 1
+ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot.sh <ip>
+
+
 [apache-modsecurity]
 
 port     = http,https

config/paths-common.conf

@@ -61,3 +61,6 @@ dovecot_log = %(syslog_mail_warn)s
 solidpop3d_log = %(syslog_local0)s
 
 mysql_log = %(syslog_daemon)s
+
+# Directory with ignorecommand scripts
+ignorecommands_dir = /etc/fail2ban/filter.d/ignorecommands

fail2ban/client/filterreader.py

@@ -71,7 +71,7 @@ class FilterReader(DefinitionInitConfigReader):
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self._jailName, "addignoreregex", regex])		
+						stream.append(["set", self._jailName, "addignoreregex", regex])
 		if self._initOpts:
 			if 'maxlines' in self._initOpts:
 				# We warn when multiline regex is used without maxlines > 1

fail2ban/tests/files/logs/apache-fakegooglebot

@@ -0,0 +1,5 @@
+# Apache 2.2
+# failJSON: { "time": "2015-01-31T14:29:44", "match": true, "host": "66.249.66.1" }
+66.249.66.1 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + 293 1149 546
+# failJSON: { "time": "2015-01-31T14:29:44", "match": false, "host": "93.184.216.34" }
+93.184.216.34 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "NOT A __GOOGLE_BOT__" + 293 1149 546

fail2ban/tests/samplestestcase.py

@@ -141,7 +141,8 @@ def testSampleRegexsFactory(name):
 
 	return testFilter
 
-for filter_ in filter(lambda x: not x.endswith('common.conf'), os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
+for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'),
+					  os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
 	filterName = filter_.rpartition(".")[0]
 	if not filterName.startswith('.'):
 		setattr(

setup.py

@@ -96,7 +96,7 @@ setup(
 		'bin/fail2ban-client',
 		'bin/fail2ban-server',
 		'bin/fail2ban-regex',
-		'bin/fail2ban-testcases',
+		'bin/fail2ban-testcases'
 	],
 	packages = [
 		'fail2ban',
@@ -124,6 +124,9 @@ setup(
 		('/etc/fail2ban/filter.d',
 			glob("config/filter.d/*.conf")
 		),
+		('/etc/fail2ban/filter.d/ignorecommands',
+			glob("config/filter.d/ignorecommands/*")
+		),
 		('/etc/fail2ban/action.d',
 			glob("config/action.d/*.conf") +
 			glob("config/action.d/*.py")