Merge branch '0.11'

ChangeLog

@@ -28,6 +28,8 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition
 * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686)
 * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), 
   so would bother the action interpolation
+* fixed type conversion in config readers (take place after all interpolations get ready), that allows to 
+  specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters.
 * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy
   between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703)
 * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing
@@ -45,6 +47,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition
 
 ### New Features and Enhancements
 * new filter and jail for GitLab recognizing failed application logins (gh-2689)
+* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
 * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
 * datetemplate: improved anchor detection for capturing groups `(^...)`;
 * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc)

config/filter.d/guacamole.conf

@@ -5,21 +5,47 @@
 
 [Definition]
 
-# Option: failregex
-# Notes.: regex to match the password failures messages in the logfile.
-# Values: TEXT
-#
+logging = catalina
+failregex = <L_<logging>/failregex>
+maxlines = <L_<logging>/maxlines>
+datepattern = <L_<logging>/datepattern>
+
+[L_catalina]
+
 failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" failed\.$
 
-# Option:  ignoreregex
-# Notes.:  regex to ignore. If this regex matches, the line is ignored.
-# Values:  TEXT
-#
-ignoreregex = 
-
-# "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
 
 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
               ^WARNING:()**
-              {^LN-BEG}
+              {^LN-BEG}
+
+[L_webapp]
+
+failregex = ^ \[\S+\] WARN  \S+ - Authentication attempt from <HOST> for user "<F-USER>[^"]+</F-USER>" failed.
+
+maxlines = 1
+
+datepattern = ^%%H:%%M:%%S.%%f
+
+# DEV Notes:
+#
+# failregex is based on the default pattern given in Guacamole documentation :
+# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging
+#
+# The following logback.xml Guacamole configuration file can then be used accordingly :
+# <configuration>
+#   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+#     <file>/var/log/guacamole.log</file>
+#     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+#       <fileNamePattern>/var/log/guacamole.%d.log.gz</fileNamePattern>
+#       <maxHistory>32</maxHistory>
+#     </rollingPolicy>
+#     <encoder>
+#       <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+#     </encoder>
+#   </appender>
+#   <root level="info">
+#     <appender-ref ref="FILE" />
+#   </root>
+# </configuration>

config/jail.conf

@@ -478,6 +478,7 @@ backend  = %(syslog_backend)s
 
 port     = http,https
 logpath  = /var/log/tomcat*/catalina.out
+#logpath  = /var/log/guacamole.log
 
 [monit]
 #Ban clients brute-forcing the monit gui login

fail2ban/client/actionreader.py

@@ -38,18 +38,18 @@ class ActionReader(DefinitionInitConfigReader):
 
 	_configOpts = {
 		"actionstart": ["string", None],
-		"actionstart_on_demand": ["string", None],
+		"actionstart_on_demand": ["bool", None],
 		"actionstop": ["string", None],
 		"actionflush": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
-		"actionrepair_on_unban": ["string", None],
+		"actionrepair_on_unban": ["bool", None],
 		"actionban": ["string", None],
 		"actionprolong": ["string", None],
 		"actionreban": ["string", None],
 		"actionunban": ["string", None],
-		"norestored": ["string", None],
+		"norestored": ["bool", None],
 	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
@@ -84,11 +84,6 @@ class ActionReader(DefinitionInitConfigReader):
 	def convert(self):
 		opts = self.getCombined(
 			ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
-		# type-convert only after combined (otherwise boolean converting prevents substitution):
-		for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'):
-			if opts.get(o):
-				opts[o] = self._convert_to_boolean(opts[o])
-		
 		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()

fail2ban/client/configparserinc.py

@@ -29,7 +29,7 @@ import re
 import sys
 from ..helpers import getLogger
 
-if sys.version_info >= (3,2):
+if sys.version_info >= (3,): # pragma: 2.x no cover
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
@@ -61,7 +61,7 @@ if sys.version_info >= (3,2):
 				return super(BasicInterpolationWithName, self)._interpolate_some(
 					parser, option, accum, rest, section, map, *args, **kwargs)
 
-else: # pragma: no cover
+else: # pragma: 3.x no cover
 	from ConfigParser import SafeConfigParser, \
 		InterpolationMissingOptionError, NoOptionError, NoSectionError
 
@@ -372,7 +372,8 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							self.merge_section('KNOWN/'+n, s2, '')
+							self.merge_section('KNOWN/'+n, 
+								dict(filter(lambda i: i[0] in s, s2.iteritems())), '')
 							# merge section
 							s2.update(s)
 						else:

fail2ban/client/configreader.py

@@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+CONVERTER = {
+	"bool": _as_bool,
+	"int": int,
+}
+def _OptionsTemplateGen(options):
+	"""Iterator over the options template with default options.
+	
+	Each options entry is composed of an array or tuple with:
+		[[type, name, ?default?], ...]
+	Or it is a dict:
+		{name: [type, default], ...}
+	"""
+	if isinstance(options, (list,tuple)):
+		for optname in options:
+			if len(optname) > 2:
+				opttype, optname, optvalue = optname
+			else:
+				(opttype, optname), optvalue = optname, None
+			yield opttype, optname, optvalue
+	else:
+		for optname in options:
+			opttype, optvalue = options[optname]
+			yield opttype, optname, optvalue
+
 
 class ConfigReader():
 	"""Generic config reader class.
@@ -228,31 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
+	def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True):
 		values = dict()
 		if pOptions is None:
 			pOptions = {}
 		# Get only specified options:
-		for optname in options:
-			if isinstance(options, (list,tuple)):
-				if len(optname) > 2:
-					opttype, optname, optvalue = optname
-				else:
-					(opttype, optname), optvalue = optname, None
-			else:
-				opttype, optvalue = options[optname]
+		for opttype, optname, optvalue in _OptionsTemplateGen(options):
 			if optname in pOptions:
 				continue
 			try:
-				if opttype == "bool":
-					v = self.getboolean(sec, optname)
-					if v is None: continue
-				elif opttype == "int":
-					v = self.getint(sec, optname)
-					if v is None: continue
-				else:
-					v = self.get(sec, optname, vars=pOptions)
+				v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v
+				if convert:
+					conv = CONVERTER.get(opttype)
+					if conv:
+						if v is None: continue
+						values[optname] = conv(v)
 			except NoSectionError as e:
 				if shouldExist:
 					raise
@@ -324,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader):
 			pOpts = dict()
 		if self._initOpts:
 			pOpts = _merge_dicts(pOpts, self._initOpts)
+		# type-convert only in combined (otherwise int/bool converting prevents substitution):
 		self._opts = ConfigReader.getOptions(
-			self, "Definition", self._configOpts, pOpts)
+			self, "Definition", self._configOpts, pOpts, convert=False)
 		self._pOpts = pOpts
 		if self.has_section("Init"):
 			# get only own options (without options from default):
@@ -346,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader):
 				if opt == '__name__' or opt in self._opts: continue
 				self._opts[opt] = self.get("Definition", opt)
 
+	def convertOptions(self, opts, configOpts):
+		"""Convert interpolated combined options to expected type.
+		"""
+		for opttype, optname, optvalue in _OptionsTemplateGen(configOpts):
+			conv = CONVERTER.get(opttype)
+			if conv:
+				v = opts.get(optname)
+				if v is None: continue
+				try:
+					opts[optname] = conv(v)
+				except ValueError:
+					logSys.warning("Wrong %s value %r for %r. Using default one: %r",
+						opttype, v, optname, optvalue)
+					opts[optname] = optvalue
 
-	def _convert_to_boolean(self, value):
-		return _as_bool(value)
-	
 	def getCombOption(self, optname):
 		"""Get combined definition option (as string) using pre-set and init
 		options as preselection (values with higher precedence as specified in section).
@@ -384,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader):
 			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
+		# convert options after all interpolations:
+		self.convertOptions(opts, self._configOpts)
 		return opts
 	
 	def convert(self):

fail2ban/helpers.py

@@ -398,8 +398,8 @@ def splitWithOptions(option):
 # tags (<tag>) in tagged options.
 #
 
-# max tag replacement count:
-MAX_TAG_REPLACE_COUNT = 10
+# max tag replacement count (considering tag X in tag Y repeat):
+MAX_TAG_REPLACE_COUNT = 25
 
 # compiled RE for tag name (replacement name) 
 TAG_CRE = re.compile(r'<([^ <>]+)>')
@@ -433,6 +433,7 @@ def substituteRecursiveTags(inptags, conditional='',
 	done = set()
 	noRecRepl = hasattr(tags, "getRawItem")
 	# repeat substitution while embedded-recursive (repFlag is True)
+	repCounts = {}
 	while True:
 		repFlag = False
 		# substitute each value:
@@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			value = orgval = uni_string(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
-			refCounts = {}
+			rplc = repCounts.get(tag, {})
 			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 			while m:
 				# found replacement tag:
@@ -454,13 +455,13 @@ def substituteRecursiveTags(inptags, conditional='',
 					m = tre_search(value, m.end())
 					continue
 				#logSys.log(5, 'found: %s' % rtag)
-				if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
+				if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
 					# recursive definitions are bad
 					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 					raise ValueError(
 						"properties contain self referencing definitions "
 						"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-						(tag, rtag, refCounts, value))
+						(tag, rtag, rplc, value))
 				repl = None
 				if conditional:
 					repl = tags.get(rtag + '?' + conditional)
@@ -480,7 +481,7 @@ def substituteRecursiveTags(inptags, conditional='',
 				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
-				refCounts[rtag] = refCounts.get(rtag, 0) + 1
+				rplc[rtag] = rplc.get(rtag, 0) + 1
 				# the next match for replace:
 				m = tre_search(value, m.start())
 			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
@@ -488,6 +489,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			if orgval != value:
 				# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 				if tre_search(value):
+					repCounts[tag] = rplc
 					repFlag = True
 				# copy return tags dict to prevent modifying of inptags:
 				if id(tags) == id(inptags):

fail2ban/server/mytime.py

@@ -121,8 +121,11 @@ class MyTime:
 
 		@return ISO-capable string representation of given unixTime
 		"""
-		return datetime.datetime.fromtimestamp(
-			unixTime).replace(microsecond=0).strftime(format)
+		# consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ)
+		dt = datetime.datetime.fromtimestamp(
+			unixTime).replace(microsecond=0
+		) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59)
+		return dt.strftime(format)
 
 	## precreate/precompile primitives used in str2seconds:
 

fail2ban/tests/actiontestcase.py

@@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase):
 		delattr(self.__action, 'ac')
 		# produce self-referencing query except:
 		self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query",
-			lambda: self.__action.replaceTag("<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x>>>>>>>>>>>>>>>>>>>>>", 
+			lambda: self.__action.replaceTag("<x"*30+">"*30,
 				self.__action._properties, conditional="family=inet6")
 		)
 

fail2ban/tests/banmanagertestcase.py

@@ -154,6 +154,21 @@ class AddFailure(unittest.TestCase):
 		finally:
 			self.__banManager.setBanTime(btime)
 
+	def testBanList(self):
+		tickets = [
+			BanTicket('192.0.2.1', 1167605999.0),
+			BanTicket('192.0.2.2', 1167605999.0),
+		]
+		tickets[1].setBanTime(-1)
+		for t in tickets:
+			self.__banManager.addBanTicket(t)
+		self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True),
+			[
+			  '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59',
+			  '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59'
+			]
+		)
+
 
 class StatusExtendedCymruInfo(unittest.TestCase):
 	def setUp(self):

fail2ban/tests/clientreadertestcase.py

@@ -87,6 +87,21 @@ option = %s
 		self.assertTrue(self.c.read(f))	# we got some now
 		return self.c.getOptions('section', [("int", 'option')])['option']
 
+	def testConvert(self):
+		self.c.add_section("Definition")
+		self.c.set("Definition", "a", "1")
+		self.c.set("Definition", "b", "1")
+		self.c.set("Definition", "c", "test")
+		opts = self.c.getOptions("Definition", 
+			(('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0)))
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0})
+		opts = self.c.getOptions("Definition", 
+			(('int', 'a'), ('bool', 'b'), ('int', 'c')))
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None})
+		opts = self.c.getOptions("Definition", 
+			{'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)})
+		self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0})
+
 	def testInaccessibleFile(self):
 		f = os.path.join(self.d, "d.conf")  # inaccessible file
 		self._write('d.conf', 0)
@@ -483,14 +498,12 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertRaises(NoSectionError, c.getOptions, 'test', {})
 
 
-class FilterReaderTest(unittest.TestCase):
-
-	def __init__(self, *args, **kwargs):
-		super(FilterReaderTest, self).__init__(*args, **kwargs)
-		self.__share_cfg = {}
+class FilterReaderTest(LogCaptureTestCase):
 
 	def testConvert(self):
-		output = [['multi-set', 'testcase01', 'addfailregex', [
+		output = [
+			['set', 'testcase01', 'maxlines', 1],
+			['multi-set', 'testcase01', 'addfailregex', [
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
@@ -512,7 +525,6 @@ class FilterReaderTest(unittest.TestCase):
 			['set', 'testcase01', 'addjournalmatch',
 				"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
 			['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
-			['set', 'testcase01', 'maxlines', 1], # Last for overide test
 		]
 		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
@@ -529,9 +541,18 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.read()
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
-		output[-1][-1] = "5"
+		output[0][-1] = 5; # maxlines = 5
 		self.assertSortedEqual(filterReader.convert(), output)
 
+	def testConvertOptions(self):
+		filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '<test>', 'test': 'X'},
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		opts = filterReader.getCombined();
+		self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines'
+		self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:")
+
 	def testFilterReaderSubstitionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {},
@@ -541,6 +562,17 @@ class FilterReaderTest(unittest.TestCase):
 		c = filterReader.convert()
 		self.assertSortedEqual(c, output)
 
+	def testFilterReaderSubstKnown(self):
+		# testcase02.conf + testcase02.local, test covering that known/option is not overridden
+		# with unmodified (not available) value of option from .local config file, so wouldn't
+		# cause self-recursion if option already has a reference to known/option in .conf file.
+		filterReader = FilterReader('testcase02', "jailname", {},
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		opts = filterReader.getCombined()
+		self.assertTrue('sshd' in opts['failregex'])
+		
 	def testFilterReaderSubstitionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'},

fail2ban/tests/files/filter.d/testcase02.conf

@@ -0,0 +1,12 @@
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = testcase-common.conf
+
+[Definition]
+
+_daemon = sshd
+__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+
+failregex = %(__prefix_line)s test

fail2ban/tests/files/filter.d/testcase02.local

@@ -0,0 +1,4 @@
+[Definition]
+
+# no options here, coverage for testFilterReaderSubstKnown:
+# avoid to overwrite known/option with unmodified (not available) value of option from .local config file

fail2ban/tests/files/logs/guacamole

@@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed.
 apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn
 # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" }
 WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed.
+
+# filterOptions: {"logging": "webapp"}
+
+# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" }
+12:57:32.907 [http-nio-8080-exec-10] WARN  o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed.