MRG: filter substition

ChangeLog

@@ -64,7 +64,8 @@ configuration before relying on it.
    * Multiline regex for Disconnecting: Too many authentication failures for
      root [preauth]\nConnection closed by 6X.XXX.XXX.XXX [preauth]
    * Replacing use of deprecated API (.warning, .assertEqual, etc)
-   * [..a648cc2] Filters can have options now too
+   * [..a648cc2] Filters can have options now too which are substituted into
+     failregex / ignoreregex
    * [..e019ab7] Multiple instances of the same action are allowed in the
      same jail -- use actname option to disambiguate.
 

MANIFEST

@@ -86,6 +86,7 @@ fail2ban/tests/files/config/apache-auth/README
 fail2ban/tests/files/config/apache-auth/noentry/.htaccess
 fail2ban/tests/files/database_v1.db
 fail2ban/tests/files/ignorecommand.py
+fail2ban/tests/files/filter.d/substition.conf
 fail2ban/tests/files/filter.d/testcase-common.conf
 fail2ban/tests/files/filter.d/testcase01.conf
 fail2ban/tests/files/testcase01.log

bin/fail2ban-regex

@@ -41,7 +41,7 @@ except ImportError:
 	journal = None
 
 from fail2ban.version import version
-from fail2ban.client.configparserinc import SafeConfigParserWithIncludes
+from fail2ban.client.filterreader import FilterReader
 from fail2ban.server.filter import Filter
 from fail2ban.server.failregex import RegexException
 
@@ -206,8 +206,6 @@ class LineStats(object):
 
 class Fail2banRegex(object):
 
-	CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"}
-
 	def __init__(self, opts):
 		self._verbose = opts.verbose
 		self._debuggex = opts.debuggex
@@ -257,46 +255,37 @@ class Fail2banRegex(object):
 		assert(regextype in ('fail', 'ignore'))
 		regex = regextype + 'regex'
 		if os.path.isfile(value):
-			reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS)
-			try:
-				reader.read(value)
-				print "Use %11s file : %s" % (regex, value)
-				# TODO: reuse functionality in client
-				regex_values = [
-					RegexStat(m)
-					for m in reader.get("Definition", regex).split('\n')
-					if m != ""]
-			except NoSectionError:
-				print "No [Definition] section in %s" % value
-				return False
-			except NoOptionError:
-				print "No %s option in %s" % (regex, value)
-				return False
-			except MissingSectionHeaderError:
-				print "No section headers in %s" % value
-				return False
+			print "Use %11s file : %s" % (regex, value)
+			reader = FilterReader(value, 'fail2ban-regex-jail', {})
+			reader.setBaseDir(None)
 
-			# Read out and set possible value of maxlines
-			try:
-				maxlines = reader.get("Init", "maxlines")
-			except (NoSectionError, NoOptionError):
-				# No [Init].maxlines found.
-				pass
+			if reader.readexplicit():
+				reader.getOptions(None)
+				readercommands = reader.convert()
+				regex_values = [
+					RegexStat(m[3])
+					for m in filter(
+						lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype,
+						readercommands)]
+				# Read out and set possible value of maxlines
+				for command in readercommands:
+					if command[2] == "maxlines":
+						maxlines = int(command[3])
+						try:
+							self.setMaxLines(maxlines)
+						except ValueError:
+							print "ERROR: Invalid value for maxlines (%(maxlines)r) " \
+								  "read from %(value)s" % locals()
+							return False
+					elif command[2] == 'addjournalmatch':
+						journalmatch = command[3]
+						self.setJournalMatch(shlex.split(journalmatch))
+					elif command[2] == 'datepattern':
+						datepattern = command[3]
+						self.setDatePattern(datepattern)
 			else:
-				try:
-					self.setMaxLines(maxlines)
-				except ValueError:
-					print "ERROR: Invalid value for maxlines (%(maxlines)r) " \
-					      "read from %(value)s" % locals()
-					return False
-			# Read out and set possible value for journalmatch
-			try:
-				journalmatch = reader.get("Init", "journalmatch")
-			except (NoSectionError, NoOptionError):
-				# No [Init].journalmatch found.
-				pass
-			else:
-				self.setJournalMatch(shlex.split(journalmatch))
+				print "ERROR: failed to read %s" % value
+				return False
 		else:
 			print "Use %11s line : %s" % (regex, shortstr(value))
 			regex_values = [RegexStat(value)]

fail2ban/client/configreader.py

@@ -158,6 +158,10 @@ class DefinitionInitConfigReader(ConfigReader):
 	
 	def read(self):
 		return ConfigReader.read(self, self._file)
+
+	# needed for fail2ban-regex that doesn't need fancy directories
+	def readexplicit(self):
+		return SafeConfigParserWithIncludes.read(self, self._file)
 	
 	def getOptions(self, pOpts):
 		self._opts = ConfigReader.getOptions(

fail2ban/client/filterreader.py

@@ -27,6 +27,7 @@ __license__ = "GPL"
 import logging, os, shlex
 
 from .configreader import ConfigReader, DefinitionInitConfigReader
+from ..server.action import CommandAction
 
 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)
@@ -43,14 +44,18 @@ class FilterReader(DefinitionInitConfigReader):
 	
 	def convert(self):
 		stream = list()
-		for opt in self._opts:
+		combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items()))
+		opts = CommandAction.substituteRecursiveTags(combinedopts)
+		if not opts:
+			raise ValueError('recursive tag definitions unable to be resolved')
+		for opt, value in opts.iteritems():
 			if opt == "failregex":
-				for regex in self._opts[opt].split('\n'):
+				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
 						stream.append(["set", self._jailName, "addfailregex", regex])
 			elif opt == "ignoreregex":
-				for regex in self._opts[opt].split('\n'):
+				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
 						stream.append(["set", self._jailName, "addignoreregex", regex])		

fail2ban/server/action.py

@@ -372,19 +372,27 @@ class CommandAction(ActionBase):
 		for tag, value in tags.iteritems():
 			value = str(value)
 			m = t.search(value)
+			done = []
+			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 			while m:
-				if m.group(1) == tag:
+				found_tag = m.group(1)
+				#logSys.log(5, 'found: %s' % found_tag)
+				if found_tag == tag or found_tag in done:
 					# recursive definitions are bad
+					#logSys.log(5, 'recursion fail')
 					return False
 				else:
-					if tags.has_key(m.group(1)):
-						value = value[0:m.start()] + tags[m.group(1)] + value[m.end():]
+					if tags.has_key(found_tag):
+						value = value[0:m.start()] + tags[found_tag] + value[m.end():]
+						#logSys.log(5, 'value now: %s' % value)
+						done.append(found_tag)
 						m = t.search(value, m.start())
 					else:
 						# Missing tags are ok so we just continue on searching.
 						# cInfo can contain aInfo elements like <HOST> and valid shell
 						# constructs like <STDIN>.
 						m = t.search(value, m.start() + 1)
+			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 			tags[tag] = value
 		return tags
 

fail2ban/tests/actiontestcase.py

@@ -53,6 +53,9 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<A>'}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
+		# Unresolveable substition
+		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
+		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})

fail2ban/tests/clientreadertestcase.py

@@ -311,6 +311,34 @@ class FilterReaderTest(unittest.TestCase):
 		output[-1][-1] = "5"
 		self.assertEqual(sorted(filterReader.convert()), sorted(output))
 
+
+	def testFilterReaderSubstitionDefault(self):
+		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
+		filterReader = FilterReader('substition', "jailname", {})
+		filterReader.setBaseDir(TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		c = filterReader.convert()
+		self.assertEqual(sorted(c), sorted(output))
+
+	def testFilterReaderSubstitionSet(self):
+		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
+		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'})
+		filterReader.setBaseDir(TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		c = filterReader.convert()
+		self.assertEqual(sorted(c), sorted(output))
+
+	def testFilterReaderSubstitionFail(self):
+		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
+		filterReader = FilterReader('substition', "jailname", {'honeypot': '<sweet>', 'sweet': '<honeypot>'})
+		filterReader.setBaseDir(TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		self.assertRaises(ValueError, FilterReader.convert, filterReader)
+
+
 class JailsReaderTest(LogCaptureTestCase):
 
 	def testProvidingBadBasedir(self):

fail2ban/tests/files/filter.d/substition.conf

@@ -0,0 +1,8 @@
+
+[Definition]
+
+failregex = to=<honeypot> fromip=<IP>
+
+[Init]
+
+honeypot = sweet@example.com