Merge branch '0.10' into 0.11

# Conflicts (resolved):
#	fail2ban/client/jailreader.py

config/filter.d/znc-adminlog.conf

@@ -3,12 +3,28 @@
 # to use this module, enable the adminlog module from within ZNC and point
 # logpath to its logfile (e.g. /var/lib/znc/moddata/adminlog/znc.log).
 
+[DEFAULT]
+
+logtype = file
+
 [Definition]
 
-failregex = ^\[\] \[[^]]+\] failed to login from <ADDR>$
+_daemon = znc
+
+# Prefix for different logtype (file, journal):
+#
+__prefix_file = (?:\[\]\s+)?
+__prefix_short = (?:\S+\s+%(_daemon)s\[\d+\]:)\s+
+__prefix_journal = %(__prefix_short)s
+
+__prefix_line = <__prefix_<logtype>>
+
+failregex = ^%(__prefix_line)s\[[^]]+\] failed to login from <ADDR>
 
 ignoreregex = 
 
+journalmatch = _SYSTEMD_UNIT=znc.service + _COMM=znc
+
 # DEV Notes:
 # Log format is: [<DATE+TIME>] [<USERNAME>] <ACTION> from <ADDR>
 # [2018-10-27 01:40:17] [girst] connected to ZNC from 1.2.3.4

fail2ban/client/filterreader.py

@@ -39,7 +39,7 @@ class FilterReader(DefinitionInitConfigReader):
 	_configOpts = {
 		"prefregex": ["string", None],
 		"ignoreregex": ["string", None],
-		"failregex": ["string", ""],
+		"failregex": ["string", None],
 		"maxlines": ["int", None],
 		"datepattern": ["string", None],
 		"journalmatch": ["string", None],
@@ -57,6 +57,10 @@ class FilterReader(DefinitionInitConfigReader):
 		opts = self.getCombined()
 		if not len(opts):
 			return stream
+		return FilterReader._fillStream(stream, opts, self._jailName)
+
+	@staticmethod
+	def _fillStream(stream, opts, jailName):
 		for opt, value in opts.iteritems():
 			if opt in ("failregex", "ignoreregex"):
 				if value is None: continue
@@ -66,21 +70,20 @@ class FilterReader(DefinitionInitConfigReader):
 					if regex != '':
 						multi.append(regex)
 				if len(multi) > 1:
-					stream.append(["multi-set", self._jailName, "add" + opt, multi])
+					stream.append(["multi-set", jailName, "add" + opt, multi])
 				elif len(multi):
-					stream.append(["set", self._jailName, "add" + opt, multi[0]])
+					stream.append(["set", jailName, "add" + opt, multi[0]])
 			elif opt in ('maxlines', 'prefregex'):
 				# Be sure we set this options first.
-				stream.insert(0, ["set", self._jailName, opt, value])
+				stream.insert(0, ["set", jailName, opt, value])
 			elif opt in ('datepattern'):
-				stream.append(["set", self._jailName, opt, value])
+				stream.append(["set", jailName, opt, value])
-			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
+				# Do not send a command if the match is empty.
 				if value is None: continue
 				for match in value.split("\n"):
 					if match == '': continue
 					stream.append(
-						["set", self._jailName, "addjournalmatch"] +
+						["set", jailName, "addjournalmatch"] + shlex.split(match))
-                        shlex.split(match))
 		return stream
 		

fail2ban/client/jailreader.py

@@ -86,36 +86,39 @@ class JailReader(ConfigReader):
 				logSys.warning("File %s is a dangling link, thus cannot be monitored" % p)
 		return pathList
 
+	_configOpts1st = {
+		"enabled": ["bool", False],
+		"backend": ["string", "auto"],
+		"filter": ["string", ""]
+	}
+	_configOpts = {
+		"enabled": ["bool", False],
+		"backend": ["string", "auto"],
+		"maxretry": ["int", None],
+		"maxmatches": ["int", None],
+		"findtime": ["string", None],
+		"bantime": ["string", None],
+		"bantime.increment": ["bool", None],
+		"bantime.factor": ["string", None],
+		"bantime.formula": ["string", None],
+		"bantime.multipliers": ["string", None],
+		"bantime.maxtime": ["string", None],
+		"bantime.rndtime": ["string", None],
+		"bantime.overalljails": ["bool", None],
+		"usedns": ["string", None], # be sure usedns is before all regex(s) in stream
+		"ignorecommand": ["string", None],
+		"ignoreself": ["bool", None],
+		"ignoreip": ["string", None],
+		"ignorecache": ["string", None],
+		"filter": ["string", ""],
+		"logtimezone": ["string", None],
+		"logencoding": ["string", None],
+		"logpath": ["string", None], # logpath after all log-related data (backend, date-pattern, etc)
+		"action": ["string", ""]
+	}
+	_configOpts.update(FilterReader._configOpts)
+
 	def getOptions(self):
-		opts1st = [["bool", "enabled", False],
-				["string", "backend", "auto"],
-				["string", "filter", ""]]
-		opts = [["bool", "enabled", False],
-				["string", "backend", "auto"],
-				["int",    "maxretry", None],
-				["int",    "maxmatches", None],
-				["string", "findtime", None],
-				["string", "bantime", None],
-				["bool",   "bantime.increment", None],
-				["string", "bantime.factor", None],
-				["string", "bantime.formula", None],
-				["string", "bantime.multipliers", None],
-				["string", "bantime.maxtime", None],
-				["string", "bantime.rndtime", None],
-				["bool",   "bantime.overalljails", None],
-				["string", "usedns", None], # be sure usedns is before all regex(s) in stream
-				["string", "failregex", None],
-				["string", "ignoreregex", None],
-				["string", "ignorecommand", None],
-				["bool",   "ignoreself", None],
-				["string", "ignoreip", None],
-				["string", "ignorecache", None],
-				["string", "filter", ""],
-				["string", "datepattern", None],
-				["string", "logtimezone", None],
-				["string", "logencoding", None],
-				["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc)
-				["string", "action", ""]]
 
 		# Before interpolation (substitution) add static options always available as default:
 		self.merge_defaults({
@@ -125,7 +128,8 @@ class JailReader(ConfigReader):
 		try:
 
 			# Read first options only needed for merge defaults ('known/...' from filter):
-			self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True)
+			self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts1st,
+				shouldExist=True)
 			if not self.__opts: # pragma: no cover
 				raise JailDefError("Init jail options failed")
 		
@@ -157,7 +161,7 @@ class JailReader(ConfigReader):
 				logSys.warning("No filter set for jail %s" % self.__name)
 
 			# Read second all options (so variables like %(known/param) can be interpolated):
-			self.__opts = ConfigReader.getOptions(self, self.__name, opts)
+			self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts)
 			if not self.__opts: # pragma: no cover
 				raise JailDefError("Read jail options failed")
 		
@@ -234,8 +238,11 @@ class JailReader(ConfigReader):
 		if e:
 			stream.extend([['config-error', "Jail '%s' skipped, because of wrong configuration: %s" % (self.__name, e)]])
 			return stream
+		# fill jail with filter options, using filter (only not overriden in jail):
 		if self.__filter:
 			stream.extend(self.__filter.convert())
+		# and using options from jail:
+		FilterReader._fillStream(stream, self.__opts, self.__name)
 		for opt, value in self.__opts.iteritems():
 			if opt == "logpath":
 				if self.__opts.get('backend', '').startswith("systemd"): continue
@@ -262,17 +269,8 @@ class JailReader(ConfigReader):
 				backend = value
 			elif opt == "ignoreip":
 				stream.append(["set", self.__name, "addignoreip"] + splitwords(value))
-			elif opt in ("failregex", "ignoreregex"):
+			elif (opt not in ('action', 'filter', 'enabled')
-				multi = []
+				and opt not in FilterReader._configOpts):
-				for regex in value.split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						multi.append(regex)
-				if len(multi) > 1:
-					stream.append(["multi-set", self.__name, "add" + opt, multi])
-				elif len(multi):
-					stream.append(["set", self.__name, "add" + opt, multi[0]])
-			elif opt not in ('action', 'filter', 'enabled'):
 				stream.append(["set", self.__name, opt, value])
 		for action in self.__actions:
 			if isinstance(action, (ConfigReaderUnshared, ConfigReader)):

fail2ban/tests/clientreadertestcase.py

@@ -302,6 +302,25 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(jail.getName(), 'sshd')
 		jail.setName('ssh-funky-blocker')
 		self.assertEqual(jail.getName(), 'ssh-funky-blocker')
+
+	def testOverrideFilterOptInJail(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf
+		jail = JailReader('sshd-override-flt-opts', basedir=IMPERFECT_CONFIG,
+			share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True)
+		self.assertTrue(jail.read())
+		self.assertTrue(jail.getOptions())
+		self.assertTrue(jail.isEnabled())
+		stream = jail.convert()
+		# check filter options are overriden with values specified directly in jail:
+		# prefregex:
+		self.assertEqual([['set', 'sshd-override-flt-opts', 'prefregex', '^Test']],
+			[o for o in stream if len(o) > 2 and o[2] == 'prefregex'])
+		# journalmatch:
+		self.assertEqual([['set', 'sshd-override-flt-opts', 'addjournalmatch', '_COMM=test']],
+			[o for o in stream if len(o) > 2 and o[2] == 'addjournalmatch'])
+		# maxlines:
+		self.assertEqual([['set', 'sshd-override-flt-opts', 'maxlines', 2]],
+			[o for o in stream if len(o) > 2 and o[2] == 'maxlines'])
 		
 	def testSplitOption(self):
 		# Simple example

fail2ban/tests/config/jail.conf

@@ -62,4 +62,13 @@ log2nd  = %(logpath)s
           d.log
 action = action[actname='ban']
          action[actname='log', logpath="%(log2nd)s"]
-         action[actname='test']
+         action[actname='test']
+
+[sshd-override-flt-opts]
+filter = zzz-sshd-obsolete-multiline[logtype=short]
+backend = systemd
+prefregex = ^Test
+failregex = ^Test unused <ADDR>$
+journalmatch = _COMM=test
+maxlines = 2
+enabled = false

fail2ban/tests/files/logs/murmur

@@ -3,3 +3,8 @@
 
 # failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" }
 <W>2015-11-29 17:18:20.962 1 => <8:testUsernameTwo(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user
+
+# filterOptions: {"logtype": "journal"}
+
+# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" }
+Test murmurd[2064]: <W>2019-09-08 13:00:05.615 1 => <10:Test(-1)> Rejected connection from 192.0.2.1:31752: Invalid server password

fail2ban/tests/files/logs/znc-adminlog

@@ -5,3 +5,11 @@
 [2018-10-27 01:40:17] [girst] connected to ZNC from 1.2.3.4
 # failJSON: { "match": false }
 [2018-10-27 01:40:21] [girst] disconnected from ZNC from 1.2.3.4
+
+# failJSON: { "time": "2019-09-08T15:53:19", "match": true , "host": "192.0.2.1", "desc": "port after IP" }
+[2019-09-08 15:53:19] [admin] failed to login from 192.0.2.1:65001
+
+# filterOptions: {"logtype": "journal"}
+
+# failJSON: { "match": true , "host": "192.0.2.2", "desc": "systemd-journal entry, port after IP" }
+Test znc[37232]: [admin] failed to login from 192.0.2.2:65009