Merge 'upstream/master' into ban-time-incr:

Merge remote-tracking branch 'sebres:cache-config-read-820' into ban-time-incr:
config cache optimized - prevent to read the same config file inside different resources multiple times;
test case: read jail file only once;
+ optimized merge: use OrderedDict.update instead of merge in cycle;

ChangeLog

@@ -71,6 +71,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger
    - New filters:
      - monit  Thanks Jason H Martin
      - directadmin  Thanks niorg
+     - apache-shellshock  Thanks Eugene Hopkinson (SlowRiot)
    - New actions:
      - symbiosis-blacklist-allports  for Bytemark symbiosis firewall
    - fail2ban-client can fetch the running server version

THANKS

@@ -34,6 +34,7 @@ David Nutter
 Derek Atkins
 Eric Gerbier
 Enrico Labedzki
+Eugene Hopkinson (SlowRiot)
 ftoppi
 François Boulogne
 Frédéric

config/filter.d/apache-shellshock.conf

@@ -0,0 +1,26 @@
+# Fail2Ban filter to block web requests containing custom headers attempting to exploit the shellshock bug
+#
+#
+
+[INCLUDES]
+
+# overwrite with apache-common.local if _apache_error_client is incorrect.
+before = apache-common.conf
+
+[Definition]
+
+failregex = ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: warning: HTTP_.*?: ignoring function definition attempt(, referer: \S+)?\s*$
+            ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: error importing function definition for `HTTP_.*?'(, referer: \S+)?\s*$
+
+ignoreregex = 
+
+
+# DEV Notes:
+#
+# https://wiki.apache.org/httpd/ListOfErrors for apache error IDs
+#
+# example log lines: 
+# [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
+# [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'
+#
+# Author: Eugene Hopkinson (riot@riot.so)

config/jail.conf

@@ -321,6 +321,11 @@ port     = http,https
 logpath  = %(apache_error_log)s
 maxretry = 2
 
+[apache-shellshock]
+
+port    = http,https
+logpath = $(apache_error_log)s
+maxretry = 1
 
 [nginx-http-auth]
 

fail2ban/client/configparserinc.py

@@ -124,7 +124,7 @@ class SafeConfigParserWithIncludes(object):
 		else:
 			fileNamesFull = resource
 		# check cache
-		hashv = '///'.join(fileNamesFull)
+		hashv = '\x01'.join(fileNamesFull)
 		cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0))
 		curmt = SCPWI._resource_mtime(fileNamesFull)
 		if cr is not None and mtime == curmt:
@@ -232,11 +232,42 @@ after = 1.conf
 			super(_SafeConfigParserWithIncludes, self).__init__(
 				*args, **kwargs)
 
+	def get_defaults(self):
+		return self._defaults
+
+	def get_sections(self):
+		return self._sections
 
 	def read(self, filenames):
 		if not isinstance(filenames, list):
 			filenames = [ filenames ]
-		logSys.debug("Reading files: %s", filenames)
+		if len(filenames) > 1:
+			# read multiple configs:
+			ret = []
+			alld = self.get_defaults()
+			alls = self.get_sections()
+			for filename in filenames:
+				# read single one, add to return list:
+				cfg = SafeConfigParserWithIncludes()
+				i = cfg.read(filename, get_includes=False)
+				if i:
+					ret += i
+					# merge defaults and all sections to self:
+					alld.update(cfg.get_defaults())
+					for n, s in cfg.get_sections().iteritems():
+						if isinstance(s, dict):
+							s2 = alls.get(n)
+							if isinstance(s2, dict):
+								s2.update(s)
+							else:
+								alls[n] = s.copy()
+						else:
+							alls[n] = s
+
+			return ret
+
+		# read one config :
+		logSys.debug("Reading file: %s", filenames[0])
 		if sys.version_info >= (3,2): # pragma: no cover
 			return SafeConfigParser.read(self, filenames, encoding='utf-8')
 		else:

fail2ban/tests/action_d/test_badips.py

@@ -22,11 +22,7 @@ import unittest
 import sys
 
 from ..dummyjail import DummyJail
-
+from ..utils import CONFIG_DIR
-if os.path.exists('config/fail2ban.conf'):
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
 
 if sys.version_info >= (2,7):
 	class BadIPsActionTest(unittest.TestCase):

fail2ban/tests/action_d/test_smtp.py

@@ -30,10 +30,7 @@ else:
 
 from ..dummyjail import DummyJail
 
-if os.path.exists('config/fail2ban.conf'):
+from ..utils import CONFIG_DIR
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
 
 class TestSMTPServer(smtpd.SMTPServer):
 

fail2ban/tests/clientreadertestcase.py

@@ -32,8 +32,10 @@ from ..client.configurator import Configurator
 from .utils import LogCaptureTestCase
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
+
+from .utils import CONFIG_DIR
+
 STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
-CONFIG_DIR='config' if STOCK else '/etc/fail2ban'
 
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
 
@@ -363,12 +365,22 @@ class JailsReaderTestCache(LogCaptureTestCase):
 			self.assertTrue(configurator.getOptions(None))
 			cnt = 0
 			for s in self.getLog().rsplit('\n'):
-				if re.match(r"^Reading files: .*jail.local", s):
+				if re.match(r"^Reading files?: .*jail.local", s):
 					cnt += 1
-			# if cnt > 2:
+			# if cnt > 1:
 			# 	self.printLog()
-			self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt)
+			self.assertFalse(cnt > 1, "Too many times reading of config files, cnt = %s" % cnt)
-			self.assertFalse(cnt <= 0)
+			self.assertFalse(cnt == 0)
+
+			# read whole configuration like a file2ban-client again ...
+			configurator = Configurator()
+			configurator.setBaseDir(basedir)
+			configurator.readEarly()
+			configurator.getEarlyOptions()
+			configurator.readAll()
+			self.assertTrue(configurator.getOptions(None))
+			self.assertFalse(cnt == 0)
+
 		finally:
 			shutil.rmtree(basedir)
 

fail2ban/tests/files/logs/apache-shellshock

@@ -0,0 +1,4 @@
+# failJSON: { "time": "2014-09-25T09:27:18", "match": true , "host": "89.207.132.76" }
+[Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
+# failJSON: { "time": "2014-09-25T09:29:56", "match": true , "host": "162.247.73.206" }
+[Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'

fail2ban/tests/samplestestcase.py

@@ -32,13 +32,9 @@ else:
 
 from ..server.filter import Filter
 from ..client.filterreader import FilterReader
-from .utils import setUpMyTime, tearDownMyTime
+from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
-if os.path.exists('config/fail2ban.conf'):
-	CONFIG_DIR = "config"
-else:
-	CONFIG_DIR='/etc/fail2ban'
 
 class FilterSamplesRegex(unittest.TestCase):
 

fail2ban/tests/utils.py

@@ -34,6 +34,15 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
+CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None)
+
+if not CONFIG_DIR:
+# Use heuristic to figure out where configuration files are
+	if os.path.exists(os.path.join('config','fail2ban.conf')):
+		CONFIG_DIR = 'config'
+	else:
+		CONFIG_DIR = '/etc/fail2ban'
+
 def mtimesleep():
 	# no sleep now should be necessary since polling tracks now not only
 	# mtime but also ino and size