normalization of DefinitionInitConfigReader (action / filter): client-side interpolation, etc.

fail2ban/client/actionreader.py

@@ -43,10 +43,15 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionrepair": ["string", None],
 		"actionban": ["string", None],
 		"actionunban": ["string", None],
+		"norestored": ["string", None],
 	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
-		self._name = initOpts.get("actname", file_)
+		actname = initOpts.get("actname")
+		if actname is None:
+			actname = file_
+			initOpts["actname"] = actname
+		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
 
@@ -64,16 +69,22 @@ class ActionReader(DefinitionInitConfigReader):
 		return self._name
 
 	def convert(self):
+		opts = self.getCombined(ignore=('timeout', 'bantime'))
+		# type-convert only after combined (otherwise boolean converting prevents substitution):
+		if opts.get('norestored'):
+			opts['norestored'] = self._convert_to_boolean(opts['norestored'])
+		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
 		multi = []
-		for opt, optval in self._opts.iteritems():
+		for opt, optval in opts.iteritems():
 			if opt in self._configOpts:
 				multi.append([opt, optval])
 		if self._initOpts:
 			for opt, optval in self._initOpts.iteritems():
-				multi.append([opt, optval])
+				if opt not in self._configOpts:
+					multi.append([opt, optval])
 		if len(multi) > 1:
 			stream.append(["multi-set", self._jailName, "action", self._name, multi])
 		elif len(multi):

fail2ban/client/configreader.py

@@ -30,6 +30,7 @@ from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
+from ..server.action import CommandAction
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -320,5 +321,27 @@ class DefinitionInitConfigReader(ConfigReader):
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
 
+	def _convert_to_boolean(self, value):
+		return value.lower() in ("1", "yes", "true", "on")
+	
+	def getCombined(self, ignore=()):
+		combinedopts = self._opts
+		ignore = set(ignore).copy()
+		if self._initOpts:
+			combinedopts = _merge_dicts(self._opts, self._initOpts)
+		if not len(combinedopts):
+			return {}
+		# ignore conditional options:
+		for n in combinedopts:
+			cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+			if cond:
+				n, cond = cond.groups()
+				ignore.add(n)
+		# substiture options already specified direct:
+		opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore)
+		if not opts:
+			raise ValueError('recursive tag definitions unable to be resolved')
+		return opts
+	
 	def convert(self):
 		raise NotImplementedError

fail2ban/client/filterreader.py

@@ -27,8 +27,7 @@ __license__ = "GPL"
 import os
 import shlex
 
-from .configreader import DefinitionInitConfigReader, _merge_dicts
+from .configreader import DefinitionInitConfigReader
-from ..server.action import CommandAction
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -52,17 +51,6 @@ class FilterReader(DefinitionInitConfigReader):
 	def getFile(self):
 		return self.__file
 
-	def getCombined(self):
-		combinedopts = self._opts
-		if self._initOpts:
-			combinedopts = _merge_dicts(self._opts, self._initOpts)
-		if not len(combinedopts):
-			return {}
-		opts = CommandAction.substituteRecursiveTags(combinedopts)
-		if not opts:
-			raise ValueError('recursive tag definitions unable to be resolved')
-		return opts
-	
 	def convert(self):
 		stream = list()
 		opts = self.getCombined()
@@ -70,6 +58,7 @@ class FilterReader(DefinitionInitConfigReader):
 			return stream
 		for opt, value in opts.iteritems():
 			if opt in ("failregex", "ignoreregex"):
+				if value is None: continue
 				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
@@ -87,6 +76,7 @@ class FilterReader(DefinitionInitConfigReader):
 				stream.append(["set", self._jailName, "datepattern", value])
 			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
+				if value is None: continue
 				for match in value.split("\n"):
 					if match == '': continue
 					stream.append(

fail2ban/client/jailreader.py

@@ -136,7 +136,8 @@ class JailReader(ConfigReader):
 				if not filterName:
 					raise JailDefError("Invalid filter definition %r" % flt)
 				self.__filter = FilterReader(
-					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
+					filterName, self.__name, filterOpt, 
+					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				# merge options from filter as 'known/...':
 				self.__filter.getOptions(self.__opts)

fail2ban/server/action.py

@@ -365,7 +365,7 @@ class CommandAction(ActionBase):
 		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
-	def substituteRecursiveTags(cls, inptags, conditional=''):
+	def substituteRecursiveTags(cls, inptags, conditional='', ignore=()):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -387,21 +387,26 @@ class CommandAction(ActionBase):
 		# copy return tags dict to prevent modifying of inptags:
 		tags = inptags.copy()
 		t = TAG_CRE
+		ignore = set(ignore)
+		done = cls._escapedTags.copy() | ignore
 		# repeat substitution while embedded-recursive (repFlag is True)
-		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				# ignore escaped or already done:
+				# ignore escaped or already done (or in ignore list):
 				if tag in done: continue
-				value = str(tags[tag])
+				value = orgval = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
 				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
+					# don't replace tags that should be currently ignored (pre-replacement):
+					if found_tag in ignore: 
+						m = t.search(value, m.end())
+						continue
 					#logSys.log(5, 'found: %s' % found_tag)
 					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
@@ -429,7 +434,7 @@ class CommandAction(ActionBase):
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
-				if tags[tag] != value:
+				if orgval != value:
 					# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 					if t.search(value):
 						repFlag = True

fail2ban/tests/clientreadertestcase.py

@@ -347,7 +347,7 @@ class FilterReaderTest(unittest.TestCase):
 			['set', 'testcase01', 'addjournalmatch',
 				"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
 			['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
-			['set', 'testcase01', 'maxlines', "1"], # Last for overide test
+			['set', 'testcase01', 'maxlines', 1], # Last for overide test
 		]
 		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
@@ -517,12 +517,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['add', 'brokenaction', 'auto'],
 			 ['set', 'brokenaction', 'addfailregex', '<IP>'],
 			 ['set', 'brokenaction', 'addaction', 'brokenaction'],
-			 ['set',
+			 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
-			  'brokenaction',
+				 ['actionban', 'hit with big stick <ip>'],
-			  'action',
+				 ['actname', 'brokenaction']
-			  'brokenaction',
+			 ]],
-			  'actionban',
-			  'hit with big stick <ip>'],
 			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
 			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
 			 ['start', 'emptyaction'],