Merge pull request #1750 from sebres/0.10-fix-default-backend

fixes default backend handling (as default used value of `%(default/backend)s`...

ChangeLog

@@ -134,6 +134,13 @@ TODO: implementing of options resp. other tasks from PR #1346
         Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone 
 	abbreviations.
 * `filter.d/courier-auth.conf`: support failed logins with method only
+* Config reader's: introduced new syntax `%(section/option)s`, in opposite to extended interpolation of
+  python 3 `${section:option}` work with all supported python version in fail2ban and this syntax is 
+  like our another features like `%(known/option)s`, etc. (gh-1750)
+* Variable `default_backend` switched to `%(default/backend)s`, so totally backwards compatible now,
+  but now the setting of parameter `backend` in default section of `jail.local` can overwrite default
+  backend also (see gh-1750). In the future versions parameter `default_backend` can be removed (incompatibility, 
+  possibly some distributions affected).
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

config/paths-common.conf

@@ -7,7 +7,7 @@ after  = paths-overrides.local
 
 [DEFAULT]
 
-default_backend = auto
+default_backend = %(default/backend)s
 
 sshd_log = %(syslog_authpriv)s
 sshd_backend = %(default_backend)s

fail2ban/client/configparserinc.py

@@ -32,8 +32,8 @@ from ..helpers import getLogger
 if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
-	from configparser import ConfigParser as SafeConfigParser, NoSectionError, \
+	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
-		BasicInterpolation
+		InterpolationMissingOptionError, NoSectionError
 
 	# And interpolation of __name__ was simply removed, thus we need to
 	# decorate default interpolator to handle it
@@ -52,20 +52,32 @@ if sys.version_info >= (3,2):
 		But should be fine to reincarnate for our use case
 		"""
 		def _interpolate_some(self, parser, option, accum, rest, section, map,
-							  depth):
+							  *args, **kwargs):
 			if section and not (__name__ in map):
 				map = map.copy()		  # just to be safe
 				map['__name__'] = section
+				# try to wrap section options like %(section/option)s:
+				parser._map_section_options(section, option, rest, map)
 				return super(BasicInterpolationWithName, self)._interpolate_some(
-				parser, option, accum, rest, section, map, depth)
+					parser, option, accum, rest, section, map, *args, **kwargs)
 
 else: # pragma: no cover
-	from ConfigParser import SafeConfigParser, NoSectionError
+	from ConfigParser import SafeConfigParser, \
+		InterpolationMissingOptionError, NoSectionError
+
+	# Interpolate missing known/option as option from default section
+	SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some
+	def _interpolate_some(self, option, accum, rest, section, map, *args, **kwargs):
+		# try to wrap section options like %(section/option)s:
+		self._map_section_options(section, option, rest, map)
+		return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
+	SafeConfigParser._interpolate_some = _interpolate_some
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 logLevel = 7
 
+
 __all__ = ['SafeConfigParserWithIncludes']
 
 
@@ -100,6 +112,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s')
+
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
 
 	if sys.version_info >= (3,2):
@@ -117,6 +131,46 @@ after = 1.conf
 			SafeConfigParser.__init__(self, *args, **kwargs)
 			self._cfg_share = share_config
 
+	def _map_section_options(self, section, option, rest, map):
+		"""
+		Interpolates values of the section options (name syntax `%(section/option)s`).
+
+		Fallback: try to wrap missing default options as "default/options" resp. "known/options"
+		"""
+		if '/' not in rest or '%(' not in rest: # pragma: no cover
+			return 0
+		soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest)
+		if not soptrep: # pragma: no cover
+			return 0
+		for sopt, opt in soptrep:
+			if sopt not in map:
+				sec = sopt[:~len(opt)]
+				seclwr = sec.lower()
+				if seclwr != 'default':
+					if seclwr == 'known':
+						# try get raw value from known options:
+						try:
+							v = self._sections['KNOWN'][opt]
+						except KeyError:
+							# fallback to default:
+							try:
+								v = self._defaults[opt]
+							except KeyError: # pragma: no cover
+								continue
+					else:
+						# get raw value of opt in section:
+						v = self.get(sec, opt, raw=True)
+				else:
+					try:
+						v = self._defaults[opt]
+					except KeyError: # pragma: no cover
+						continue
+				self._defaults[sopt] = v
+				try: # for some python versions need to duplicate it in map-vars also:
+					map[sopt] = v
+				except: pass
+		return 1
+
 	@property
 	def share_config(self):
 		return self._cfg_share
@@ -207,7 +261,7 @@ after = 1.conf
 		"""
 		try:
 			opts = self._sections[section]
-		except KeyError:
+		except KeyError: # pragma: no cover
 			raise NoSectionError(section)
 		if withDefault:
 			# mix it with defaults:
@@ -259,11 +313,7 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							sk = {}
+							self.merge_section('KNOWN', s2, '')
-							for k, v in s2.iteritems():
-								if not k.startswith('known/') and k != '__name__':
-									sk['known/'+k] = v
-							s2.update(sk)
 							# merge section
 							s2.update(s)
 						else:
@@ -280,14 +330,18 @@ after = 1.conf
 		else:
 			return SafeConfigParser.read(self, fileNamesFull)
 
-	def merge_section(self, section, options, pref='known/'):
+	def merge_section(self, section, options, pref=None):
 		alls = self.get_sections()
-		if pref == '':
+		try:
-			alls[section].update(options)
+			sec = alls[section]
+		except KeyError:
+			alls[section] = sec = dict()
+		if not pref:
+			sec.update(options)
 			return
 		sk = {}
 		for k, v in options.iteritems():
 			if not k.startswith(pref) and k != '__name__':
 				sk[pref+k] = v
-		alls[section].update(sk)
+		sec.update(sk)
 

fail2ban/client/configreader.py

@@ -110,7 +110,7 @@ class ConfigReader():
 
 	def sections(self):
 		try:
-			return self._cfg.sections()
+			return (n for n in self._cfg.sections() if n != 'KNOWN')
 		except AttributeError:
 			return []
 

fail2ban/tests/clientreadertestcase.py

@@ -162,6 +162,40 @@ c = d ;in line comment
 		self.assertEqual(self.c.get('DEFAULT', 'b'), 'a')
 		self.assertEqual(self.c.get('DEFAULT', 'c'), 'd')
 
+	def testTargetedSectionOptions(self):
+		self.assertFalse(self.c.read('g'))	# nothing is there yet
+		self._write("g.conf", value=None, content="""
+[DEFAULT]
+a = def-a
+b = def-b,a:`%(a)s`
+c = def-c,b:"%(b)s"
+d = def-d-b:"%(known/b)s"
+
+[jail]
+a = jail-a-%(test/a)s
+b = jail-b-%(test/b)s
+y = %(test/y)s
+
+[test]
+a = test-a-%(default/a)s
+b = test-b-%(known/b)s
+x = %(test/x)s
+y = %(jail/y)s
+""")
+		self.assertTrue(self.c.read('g'))
+		self.assertEqual(self.c.get('test', 'a'), 'test-a-def-a')
+		self.assertEqual(self.c.get('test', 'b'), 'test-b-def-b,a:`test-a-def-a`')
+		self.assertEqual(self.c.get('jail', 'a'), 'jail-a-test-a-def-a')
+		self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`jail-a-test-a-def-a`')
+		self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"')
+		self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"')
+		self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('test', 'd'),  'def-d-b:"def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"')
+		self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"')
+		self.assertRaises(Exception, self.c.get, 'test', 'x')
+		self.assertRaises(Exception, self.c.get, 'jail', 'y')
+
 
 class JailReaderTest(LogCaptureTestCase):
 

man/jail.conf.5

@@ -90,11 +90,16 @@ indicates that the specified file is to be parsed after the current file.
 .RE
 
 Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s.
-Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g.
+
+Fail2ban has more advanced syntax (similar python extended interpolation). This extended interpolation is using \fB%(section/parameter)s\fR to denote a value from a foreign section.
+.br
+Besides cross section interpolation the value of parameter in \fI[DEFAULT]\fR section can be retrieved with \fB%(default/parameter)s\fR.
+.br
+Fail2ban supports also another feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g.
 
 .RS
 .nf
-baduseragents = IE|wget
+baduseragents = IE|wget|%(my-settings/baduseragents)s
 failregex = %(known/failregex)s
             useragent=%(baduseragents)s
 .fi