Merge: 'github_kwirk_fail2ban/filter-init' into 0.9 -- allow options for filters (move maxlines into filters handling)

* github_kwirk_fail2ban/filter-init:
  ENH: Renamed OptionConfigReader to DefinitionInitConfigReader
  ENH: Rename splitAction to extractOptions in jailreader
  ENH: Use os.path.join for filter/action config readers
  ENH: Remove redundant `maxlines` option from jail reader
  TST: Add test for FilterReader [Init] `maxlines` override
  ENH: Move jail `maxlines` to filter config
  NF: Filters now allow adding of [Init] section similar to actions

Conflicts:
	fail2ban/tests/clientreadertestcase.py

config/filter.d/guacamole.conf

@@ -16,3 +16,7 @@ failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" fa
 # Values:  TEXT
 #
 ignoreregex = 
+
+[Init]
+# "maxlines" is number of log lines to buffer for multi-line regex searches
+maxlines = 2

config/jail.conf

@@ -32,9 +32,6 @@ findtime  = 600
 # "maxretry" is the number of failures before a host get banned.
 maxretry = 3
 
-# "maxlines" is number of log lines to buffer for multi-line regex searches
-maxlines = 1
-
 # "backend" specifies the backend used to get files modification.
 # Available options are "pyinotify", "gamin", "polling" and "auto".
 # This option can be overridden in each jail as well.
@@ -375,7 +372,6 @@ action   = iptables-multiport[name=Guacmole, port="http,https"]
            sendmail-whois[name=Guacamole, dest=root, sender=fail2ban@example.com]
 logpath  = /var/log/tomcat*/catalina.out
 maxretry = 5
-maxlines = 2
 
 
 # Jail for more extended banning of persistent abusers

fail2ban/client/actionreader.py

@@ -27,67 +27,43 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import logging
+import logging, os
-from configreader import ConfigReader
+from configreader import ConfigReader, DefinitionInitConfigReader
 
 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)
 
-class ActionReader(ConfigReader):
+class ActionReader(DefinitionInitConfigReader):
-	
+
-	def __init__(self, action, name, **kwargs):
+	_configOpts = [
-		ConfigReader.__init__(self, **kwargs)
+		["string", "actionstart", ""],
-		self.__file = action[0]
+		["string", "actionstop", ""],
-		self.__cInfo = action[1]
+		["string", "actioncheck", ""],
-		self.__name = name
+		["string", "actionban", ""],
-	
+		["string", "actionunban", ""],
-	def setFile(self, fileName):
+	]
-		self.__file = fileName
+
-	
-	def getFile(self):
-		return self.__file
-	
-	def setName(self, name):
-		self.__name = name
-	
-	def getName(self):
-		return self.__name
-	
 	def read(self):
-		return ConfigReader.read(self, "action.d/" + self.__file)
+		return ConfigReader.read(self, os.path.join("action.d", self._file))
-	
+
-	def getOptions(self, pOpts):
-		opts = [["string", "actionstart", ""],
-				["string", "actionstop", ""],
-				["string", "actioncheck", ""],
-				["string", "actionban", ""],
-				["string", "actionunban", ""]]
-		self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts)
-		
-		if self.has_section("Init"):
-			for opt in self.options("Init"):
-				if not self.__cInfo.has_key(opt):
-					self.__cInfo[opt] = self.get("Init", opt)
-	
 	def convert(self):
-		head = ["set", self.__name]
+		head = ["set", self._name]
 		stream = list()
-		stream.append(head + ["addaction", self.__file])
+		stream.append(head + ["addaction", self._file])
-		for opt in self.__opts:
+		for opt in self._opts:
 			if opt == "actionstart":
-				stream.append(head + ["actionstart", self.__file, self.__opts[opt]])
+				stream.append(head + ["actionstart", self._file, self._opts[opt]])
 			elif opt == "actionstop":
-				stream.append(head + ["actionstop", self.__file, self.__opts[opt]])
+				stream.append(head + ["actionstop", self._file, self._opts[opt]])
 			elif opt == "actioncheck":
-				stream.append(head + ["actioncheck", self.__file, self.__opts[opt]])
+				stream.append(head + ["actioncheck", self._file, self._opts[opt]])
 			elif opt == "actionban":
-				stream.append(head + ["actionban", self.__file, self.__opts[opt]])
+				stream.append(head + ["actionban", self._file, self._opts[opt]])
 			elif opt == "actionunban":
-				stream.append(head + ["actionunban", self.__file, self.__opts[opt]])
+				stream.append(head + ["actionunban", self._file, self._opts[opt]])
 		# cInfo
-		if self.__cInfo:
+		if self._initOpts:
-			for p in self.__cInfo:
+			for p in self._initOpts:
-				stream.append(head + ["setcinfo", self.__file, p, self.__cInfo[p]])
+				stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]])
 
 		return stream
-		

fail2ban/client/configreader.py

@@ -130,3 +130,47 @@ class ConfigReader(SafeConfigParserWithIncludes):
 							"'. Using default one: '" + `option[2]` + "'")
 				values[option[1]] = option[2]
 		return values
+
+class DefinitionInitConfigReader(ConfigReader):
+	"""Config reader for files with options grouped in [Definition] and
+       [Init] sections.
+
+       Is a base class for readers of filters and actions, where definitions
+       in jails might provide custom values for options defined in [Init]
+       section.
+       """
+
+	_configOpts = []
+	
+	def __init__(self, file_, jailName, initOpts, **kwargs):
+		ConfigReader.__init__(self, **kwargs)
+		self._file = file_
+		self._name = jailName
+		self._initOpts = initOpts
+	
+	def setFile(self, fileName):
+		self._file = fileName
+	
+	def getFile(self):
+		return self.__file
+	
+	def setName(self, name):
+		self._name = name
+	
+	def getName(self):
+		return self._name
+	
+	def read(self):
+		return ConfigReader.read(self, self._file)
+	
+	def getOptions(self, pOpts):
+		self._opts = ConfigReader.getOptions(
+			self, "Definition", self._configOpts, pOpts)
+		
+		if self.has_section("Init"):
+			for opt in self.options("Init"):
+				if not self._initOpts.has_key(opt):
+					self._initOpts[opt] = self.get("Init", opt)
+	
+	def convert(self):
+		raise NotImplementedError

fail2ban/client/filterreader.py

@@ -27,51 +27,37 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import logging
+import logging, os
-from configreader import ConfigReader
+from configreader import ConfigReader, DefinitionInitConfigReader
 
 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)
 
-class FilterReader(ConfigReader):
+class FilterReader(DefinitionInitConfigReader):
-	
+
-	def __init__(self, fileName, name, **kwargs):
+	_configOpts = [
-		ConfigReader.__init__(self, **kwargs)
+		["string", "ignoreregex", ""],
-		self.__file = fileName
+		["string", "failregex", ""],
-		self.__name = name
+	]
-	
+
-	def setFile(self, fileName):
-		self.__file = fileName
-	
-	def getFile(self):
-		return self.__file
-	
-	def setName(self, name):
-		self.__name = name
-	
-	def getName(self):
-		return self.__name
-	
 	def read(self):
-		return ConfigReader.read(self, "filter.d/" + self.__file)
+		return ConfigReader.read(self, os.path.join("filter.d", self._file))
-	
-	def getOptions(self, pOpts):
-		opts = [["string", "ignoreregex", ""],
-				["string", "failregex", ""]]
-		self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts)
 	
 	def convert(self):
 		stream = list()
-		for opt in self.__opts:
+		for opt in self._opts:
 			if opt == "failregex":
-				for regex in self.__opts[opt].split('\n'):
+				for regex in self._opts[opt].split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self.__name, "addfailregex", regex])
+						stream.append(["set", self._name, "addfailregex", regex])
 			elif opt == "ignoreregex":
-				for regex in self.__opts[opt].split('\n'):
+				for regex in self._opts[opt].split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self.__name, "addignoreregex", regex])		
+						stream.append(["set", self._name, "addignoreregex", regex])		
+		if self._initOpts:
+			if 'maxlines' in self._initOpts:
+				stream.append(["set", self._name, "maxlines", self._initOpts["maxlines"]])
 		return stream
 		

fail2ban/client/jailreader.py

@@ -38,7 +38,7 @@ logSys = logging.getLogger(__name__)
 
 class JailReader(ConfigReader):
 	
-	actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
+	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
 	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
@@ -65,7 +65,6 @@ class JailReader(ConfigReader):
 				["string", "logencoding", "auto"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
-				["int", "maxlines", 1],
 				["int", "findtime", 600],
 				["int", "bantime", 600],
 				["string", "usedns", "warn"],
@@ -78,8 +77,10 @@ class JailReader(ConfigReader):
 		
 		if self.isEnabled():
 			# Read filter
-			self.__filter = FilterReader(self.__opts["filter"], self.__name,
+			filterName, filterOpt = JailReader.extractOptions(
-										 basedir=self.getBaseDir())
+				self.__opts["filter"])
+			self.__filter = FilterReader(
+				filterName, self.__name, filterOpt, basedir=self.getBaseDir())
 			ret = self.__filter.read()
 			if ret:
 				self.__filter.getOptions(self.__opts)
@@ -92,8 +93,9 @@ class JailReader(ConfigReader):
 				try:
 					if not act:			  # skip empty actions
 						continue
-					splitAct = JailReader.splitAction(act)
+					actName, actOpt = JailReader.extractOptions(act)
-					action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir())
+					action = ActionReader(
+						actName, self.__name, actOpt, basedir=self.getBaseDir())
 					ret = action.read()
 					if ret:
 						action.getOptions(self.__opts)
@@ -124,8 +126,6 @@ class JailReader(ConfigReader):
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
 				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
-			elif opt == "maxlines":
-				stream.append(["set", self.__name, "maxlines", self.__opts[opt]])
 			elif opt == "ignoreip":
 				for ip in self.__opts[opt].split():
 					# Do not send a command if the rule is empty.
@@ -151,23 +151,23 @@ class JailReader(ConfigReader):
 		return stream
 	
 	#@staticmethod
-	def splitAction(action):
+	def extractOptions(option):
-		m = JailReader.actionCRE.match(action)
+		m = JailReader.optionCRE.match(option)
 		d = dict()
 		mgroups = m.groups()
 		if len(mgroups) == 2:
-			action_name, action_opts = mgroups
+			option_name, option_opts = mgroups
 		elif len(mgroups) == 1:
-			action_name, action_opts = mgroups[0], None
+			option_name, option_opts = mgroups[0], None
 		else:
-			raise ValueError("While reading action %s we should have got up to "
+			raise ValueError("While reading option %s we should have got up to "
-							 "2 groups. Got: %r" % (action, mgroups))
+							 "2 groups. Got: %r" % (option, mgroups))
-		if not action_opts is None:
+		if not option_opts is None:
 			# Huge bad hack :( This method really sucks. TODO Reimplement it.
-			actions = ""
+			options = ""
 			escapeChar = None
 			allowComma = False
-			for c in action_opts:
+			for c in option_opts:
 				if c in ('"', "'") and not allowComma:
 					# Start
 					escapeChar = c
@@ -178,20 +178,20 @@ class JailReader(ConfigReader):
 					allowComma = False
 				else:
 					if c == ',' and allowComma:
-						actions += "<COMMA>"
+						options += "<COMMA>"
 					else:
-						actions += c
+						options += c
 			
 			# Split using ,
-			actionsSplit = actions.split(',')
+			optionsSplit = options.split(',')
 			# Replace the tag <COMMA> with ,
-			actionsSplit = [n.replace("<COMMA>", ',') for n in actionsSplit]
+			optionsSplit = [n.replace("<COMMA>", ',') for n in optionsSplit]
 			
-			for param in actionsSplit:
+			for param in optionsSplit:
 				p = param.split('=')
 				try:
 					d[p[0].strip()] = p[1].strip()
 				except IndexError:
-					logSys.error("Invalid argument %s in '%s'" % (p, action_opts))
+					logSys.error("Invalid argument %s in '%s'" % (p, option_opts))
-		return [action_name, d]
+		return [option_name, d]
-	splitAction = staticmethod(splitAction)
+	extractOptions = staticmethod(extractOptions)

fail2ban/tests/clientreadertestcase.py

@@ -114,12 +114,13 @@ class JailReaderTest(unittest.TestCase):
 		self.assertFalse(jail.isEnabled())
 		self.assertEqual(jail.getName(), 'ssh-iptables')
 
-	def testSplitAction(self):
+	def testSplitOption(self):
 		action = "mail-whois[name=SSH]"
 		expected = ['mail-whois', {'name': 'SSH'}]
-		result = JailReader.splitAction(action)
+		result = JailReader.extractOptions(action)
-		self.assertEqual(expected, result)
+		self.assertEquals(expected, result)
-		
+
+
 class FilterReaderTest(unittest.TestCase):
 
 	def testConvert(self):
@@ -141,8 +142,9 @@ class FilterReaderTest(unittest.TestCase):
 			"error: PAM: )?User not known to the\\nunderlying authentication."
 			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"],
 			['set', 'testcase01', 'addignoreregex', 
-			"^.+ john from host 192.168.1.1\\s*$"]]
+			"^.+ john from host 192.168.1.1\\s*$"],
-		filterReader = FilterReader("testcase01", "testcase01")
+			['set', 'testcase01', 'maxlines', "1"]]
+		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
 		filterReader.read()
 		#filterReader.getOptions(["failregex", "ignoreregex"])
@@ -152,6 +154,15 @@ class FilterReaderTest(unittest.TestCase):
 		# is unreliable
 		self.assertEqual(sorted(filterReader.convert()), sorted(output))
 
+		filterReader = FilterReader(
+			"testcase01", "testcase01", {'maxlines': "5"})
+		filterReader.setBaseDir(TEST_FILES_DIR)
+		filterReader.read()
+		#filterReader.getOptions(["failregex", "ignoreregex"])
+		filterReader.getOptions(None)
+		output[-1][-1] = "5"
+		self.assertEquals(sorted(filterReader.convert()), sorted(output))
+
 class JailsReaderTest(unittest.TestCase):
 
 	def testProvidingBadBasedir(self):

fail2ban/tests/files/filter.d/testcase01.conf

@@ -32,3 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro
 # Values:  TEXT
 #
 ignoreregex = ^.+ john from host 192.168.1.1\s*$
+
+[Init]
+# "maxlines" is number of log lines to buffer for multi-line regex searches
+maxlines = 1

man/jail.conf.5

@@ -140,6 +140,11 @@ Using Python "string interpolation" mechanisms, other definitions are allowed an
  baduseragents = IE|wget
  failregex = useragent=%(baduseragents)s
 
+.PP
+Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options:
+.TP
+\fBmaxlines\fR
+specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to.
 .PP
 Filters can also have a section called [INCLUDES]. This is used to read other configuration files.