Merge pull request #919 from sebres/issue/907-embedded

Issue/907 + embedded

ChangeLog

@@ -11,6 +11,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
 -----------
 
 - Fixes:
+   * infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907.
+     Thanks TonyThompson
    * port[s] typo in jail.conf/nginx-http-auth gh-913. Thanks Frederik Wagner (fnerdwq)
    * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255
    * grep'ing for IP in *mail-whois-lines.conf should now match also
@@ -28,6 +30,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
 - New Features:
    - New filter:
      - postfix-rbl  Thanks Lee Clemens
+   - New recursive embedded substitution feature added:
+     - `<<PREF>HOST>` becomes `<IPV4HOST>` for PREF=`IPV4`;
+     - `<<PREF>HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`;
    - New interpolation feature for config readers - `%(known/parameter)s`.
      (means last known option with name `parameter`). This interpolation makes
      possible to extend a stock filter or jail regexp in .local file

fail2ban/server/action.py

@@ -362,6 +362,7 @@ class CommandAction(ActionBase):
 	@classmethod
 	def substituteRecursiveTags(cls, tags):
 		"""Sort out tag definitions within other tags.
+		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
 		so:		becomes:
 		a = 3		a = 3
@@ -378,38 +379,46 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
-		t = re.compile(r'<([^ >]+)>')
+		t = re.compile(r'<([^ <>]+)>')
-		for tag in tags.iterkeys():
+		# repeat substitution while embedded-recursive (repFlag is True)
-			if tag in cls._escapedTags:
+		while True:
-				# Escaped so won't match
+			repFlag = False
-				continue
+			# substitute each value:
-			value = str(tags[tag])
+			for tag in tags.iterkeys():
-			m = t.search(value)
+				if tag in cls._escapedTags:
-			done = []
-			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
-			while m:
-				found_tag = m.group(1)
-				#logSys.log(5, 'found: %s' % found_tag)
-				if found_tag == tag or found_tag in done:
-					# recursive definitions are bad
-					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
-					return False
-				elif found_tag in cls._escapedTags:
 					# Escaped so won't match
 					continue
-				else:
+				value = str(tags[tag])
-					if tags.has_key(found_tag):
+				# search and replace all tags within value, that can be interpolated using other tags:
-						value = value.replace('<%s>' % found_tag , tags[found_tag])
+				m = t.search(value)
-						#logSys.log(5, 'value now: %s' % value)
+				done = []
-						done.append(found_tag)
+				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
-						m = t.search(value, m.start())
+				while m:
-					else:
+					found_tag = m.group(1)
-						# Missing tags are ok so we just continue on searching.
+					#logSys.log(5, 'found: %s' % found_tag)
-						# cInfo can contain aInfo elements like <HOST> and valid shell
+					if found_tag == tag or found_tag in done:
+						# recursive definitions are bad
+						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
+						return False
+					if found_tag in cls._escapedTags or not tags.has_key(found_tag):
+						# Escaped or missing tags - just continue on searching after end of match
+						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
 						# constructs like <STDIN>.
-						m = t.search(value, m.start() + 1)
+						m = t.search(value, m.end())
-			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
+						continue
-			tags[tag] = value
+					value = value.replace('<%s>' % found_tag , tags[found_tag])
+					#logSys.log(5, 'value now: %s' % value)
+					done.append(found_tag)
+					m = t.search(value, m.start())
+				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
+				# was substituted?
+				if tags[tag] != value:
+					# check still contains any tag - should be repeated (possible embedded-recursive substitution):
+					if t.search(value):
+						repFlag = True
+					tags[tag] = value
+			if not repFlag:
+				break
 		return tags
 
 	@staticmethod

fail2ban/tests/actiontestcase.py

@@ -59,6 +59,8 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
+		# Escaped tags should be ignored
+		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<matches> <B>', 'B': 'cool'}), {'A': '<matches> cool', 'B': 'cool'})
 		# Multiple stuff on same line is ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP> evilperson=<honeypot>', 'honeypot': 'pokie', 'ignoreregex': ''}),
 								{ 'failregex': "to=pokie fromip=<IP> evilperson=pokie",
@@ -71,6 +73,14 @@ class CommandActionTest(LogCaptureTestCase):
 									'ABC': '123 192.0.2.0',
 									'xyz': '890 123 192.0.2.0',
 								})
+		# obscure embedded case
+		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4'}),
+						 {'A': '<IPV4HOST>', 'PREF': 'IPV4'})
+		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}),
+						 {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'})
+		# more embedded within a string and two interpolations
+		self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A <IP<PREF>HOST> B IP<PREF> C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}),
+						 {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'})
 
 	def testReplaceTag(self):
 		aInfo = {