New interpolation feature for definition config readers - `<known/parameter>`, as extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters;

ChangeLog

@@ -26,6 +26,13 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released
      with new default variable `banaction_allports` (gh-1216)
 
 - New Features:
+   * New interpolation feature for definition config readers - `<known/parameter>`
+     (means last known init definition of filters or actions with name `parameter`).
+     This interpolation makes possible to extend a parameters of stock filter or 
+     action directly in jail inside jail.local file, without creating a separately
+     filter.d/*.local file.
+     As extension to interpolation `%(known/parameter)s`, that does not works for
+     filter and action init parameters
    * New filters:
      - openhab - domotic software authentication failure with the
        rest api and web interface (gh-1223)

fail2ban/client/configreader.py

@@ -285,8 +285,10 @@ class DefinitionInitConfigReader(ConfigReader):
 		
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
+				v = self.get("Init", opt)
+				self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
-					self._initOpts[opt] = self.get("Init", opt)
+					self._initOpts[opt] = v
 	
 	def convert(self):
 		raise NotImplementedError

fail2ban/tests/clientreadertestcase.py

@@ -165,11 +165,11 @@ class JailReaderTest(LogCaptureTestCase):
 		self.__share_cfg = {}
 
 	def testIncorrectJail(self):
-		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg)
+		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg)
 		self.assertRaises(ValueError, jail.read)
 		
 	def testJailActionEmpty(self):
-		jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg)
+		jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
 		self.assertTrue(jail.read())
 		self.assertTrue(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
@@ -177,7 +177,7 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertLogged('No actions were defined for emptyaction')
 
 	def testJailActionFilterMissing(self):
-		jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg)
+		jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
 		self.assertTrue(jail.read())
 		self.assertFalse(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
@@ -200,7 +200,7 @@ class JailReaderTest(LogCaptureTestCase):
 
 	if STOCK:
 		def testStockSSHJail(self):
-			jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm
+			jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm
 			self.assertTrue(jail.read())
 			self.assertTrue(jail.getOptions())
 			self.assertFalse(jail.isEnabled())
@@ -274,6 +274,10 @@ class JailReaderTest(LogCaptureTestCase):
 		
 class FilterReaderTest(unittest.TestCase):
 
+	def __init__(self, *args, **kwargs):
+		super(FilterReaderTest, self).__init__(*args, **kwargs)
+		self.__share_cfg = {}
+
 	def testConvert(self):
 		output = [['set', 'testcase01', 'addfailregex',
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
@@ -311,9 +315,8 @@ class FilterReaderTest(unittest.TestCase):
 		# is unreliable
 		self.assertEqual(sorted(filterReader.convert()), sorted(output))
 
-		filterReader = FilterReader(
+		filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"},
-			"testcase01", "testcase01", {'maxlines': "5"})
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
-		filterReader.setBaseDir(TEST_FILES_DIR)
 		filterReader.read()
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
@@ -322,8 +325,8 @@ class FilterReaderTest(unittest.TestCase):
 
 	def testFilterReaderSubstitionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
-		filterReader = FilterReader('substition', "jailname", {})
+		filterReader = FilterReader('substition', "jailname", {},
-		filterReader.setBaseDir(TEST_FILES_DIR)
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
@@ -331,16 +334,34 @@ class FilterReaderTest(unittest.TestCase):
 
 	def testFilterReaderSubstitionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
-		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'})
+		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'},
-		filterReader.setBaseDir(TEST_FILES_DIR)
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		c = filterReader.convert()
+		self.assertEqual(sorted(c), sorted(output))
+
+	def testFilterReaderSubstitionKnown(self):
+		output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>']]
+		filterName, filterOpt = JailReader.extractOptions(
+			'substition[honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
+		filterReader = FilterReader('substition', "jailname", filterOpt,
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
 		self.assertEqual(sorted(c), sorted(output))
 
 	def testFilterReaderSubstitionFail(self):
-		filterReader = FilterReader('substition', "jailname", {'honeypot': '<sweet>', 'sweet': '<honeypot>'})
+		# directly subst the same var :
-		filterReader.setBaseDir(TEST_FILES_DIR)
+		filterReader = FilterReader('substition', "jailname", {'honeypot': '<honeypot>'},
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		self.assertRaises(ValueError, FilterReader.convert, filterReader)
+		# cross subst the same var :
+		filterReader = FilterReader('substition', "jailname", {'honeypot': '<sweet>', 'sweet': '<honeypot>'},
+		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		self.assertRaises(ValueError, FilterReader.convert, filterReader)
@@ -508,12 +529,13 @@ class JailsReaderTest(LogCaptureTestCase):
 				if jail == 'INCLUDES':
 					continue
 				filterName = jails.get(jail, 'filter')
+				filterName, filterOpt = JailReader.extractOptions(filterName)
 				allFilters.add(filterName)
 				self.assertTrue(len(filterName))
 				# moreover we must have a file for it
 				# and it must be readable as a Filter
-				filterReader = FilterReader(filterName, jail, {})
+				filterReader = FilterReader(filterName, jail, filterOpt, 
-				filterReader.setBaseDir(CONFIG_DIR)
+					share_config=self.__share_cfg, basedir=CONFIG_DIR)
 				self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName)		  # opens fine
 				filterReader.getOptions({})	  # reads fine
 
@@ -551,7 +573,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			filters = set(os.path.splitext(os.path.split(a)[1])[0]
 				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
 					if not a.endswith('common.conf'))
-			filters_jail = set(jail.options['filter'] for jail in jails.jails)
+			# get filters of all jails (filter names without options inside filter[...])
+			filters_jail = set(
+				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
+			)
 			self.maxDiff = None
 			self.assertTrue(filters.issubset(filters_jail),
 					"More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail))

man/jail.conf.5

@@ -89,13 +89,33 @@ indicates that the specified file is to be parsed before the current file.
 indicates that the specified file is to be parsed after the current file.
 .RE
 
-Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. For example.
+Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. 
+Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example.
 
 .RS
+.nf
 baduseragents = IE|wget
+failregex = %(known/failregex)s
+            useragent=%(baduseragents)s
+.fi
 .RE
+
+Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB<known/parameter>\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example.
+
 .RS
-failregex = useragent=%(baduseragents)s
+# filter.d/test.conf:
+.nf
+[Init]
+test.method = GET
+baduseragents = IE|wget
+[Definition]
+failregex = ^%(__prefix_line)\\s+"<test.method>"\\s+test\\s+regexp\\s+-\\s+useragent=(?:<baduseragents>)
+
+# jail.local:
+[test]
+# use filter "test", overwrite method to "POST" and extend known bad agents with "badagent":
+filter = test[test.method=POST, baduseragents="badagent|<known/baduseragents>"]
+.fi
 .RE
 
 Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug.
@@ -253,7 +273,7 @@ The maximum period of time in seconds that a command can executed, before being
 Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should
 return 0, otherwise error would be logged.  Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands).  
 Tags are enclosed in <>.  All the elements of [Init] are tags that are replaced in all action commands.  Tags can be added by the
-\fBfail2ban-client\fR using the "set <JAIL> action <ACT>" command. \fB<br>\fR is a tag that is always a new line (\\n).
+\fBfail2ban-client\fR using the "set <JAIL> action <ACT>" command. \fB<br>\fR is a tag that is always a new line (\\n).     
 
 More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines
 two commands to be executed.
@@ -312,7 +332,7 @@ is the regex to identify log entries that should be ignored by Fail2Ban, even if
 
 
 .PP
-Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options:
+Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options:
 .TP
 \fBmaxlines\fR
 specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to.
@@ -327,6 +347,8 @@ Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO860
 \fBjournalmatch\fR
 specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend.
 .PP
+Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR.
+
 Filters can also have a section called [INCLUDES]. This is used to read other configuration files.
 
 .TP